00000100:00080000:1.0F:1787695764.790469:0:42821:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695764.790499:0:42821:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1787695764.790503:0:42821:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695764.790508:0:42821:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1787695764.790512:0:42821:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695764.790517:0:42821:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1787695764.790525:0:42821:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446664700. 00000020:00000040:1.0:1787695764.790531:0:42821:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=20 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000100:00000001:1.0:1787695764.790537:0:42821:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695764.790540:0:42821:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695764.790545:0:42821:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695764.790553:0:42821:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b1445678700. 00000100:00000001:1.0:1787695764.790560:0:42821:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695764.790573:0:42821:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID req@ffff8b1446664700 x1874531139519360/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695764.790585:0:42821:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695764.790590:0:42821:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695764.790598:0:42821:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007106cb5] to pc [ptlrpcd_00_00+0] req@ffff8b1446664700 x1874531139519360/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695764.790629:0:42821:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695764.790632:0:42821:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695764.790636:0:42821:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1787695764.790640:0:42821:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695764.790644:0:42821:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1787695764.790647:0:42821:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0F:1787695764.790650:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1787695764.790651:0:42821:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000001:0.0:1787695764.790661:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1787695764.790664:0:42821:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695764.790665:0:2403:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787695764.790672:0:42821:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000040:0.0:1787695764.790673:0:2403:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446664700 x1874531139519360/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1787695764.790677:0:42821:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446667480. 00000100:00000001:0.0:1787695764.790685:0:2403:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000040:1.0:1787695764.790685:0:42821:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=21 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000001:0.0:1787695764.790688:0:2403:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695764.790692:0:42821:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695764.790693:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695764.790697:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695764.790697:0:42821:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787695764.790701:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695764.790702:0:42821:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695764.790705:0:2403:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446664700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2403:1874531139519360:192.168.201.140@tcp:400:kworker.0 00000100:00000001:0.0:1787695764.790710:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000010:1.0:1787695764.790712:0:42821:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b1445678d00. 02000000:00000001:0.0:1787695764.790714:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695764.790717:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695764.790718:0:42821:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695764.790719:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1787695764.790728:0:42821:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID req@ffff8b1446667480 x1874531139519488/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:0.0:1787695764.790732:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b1451d64e00. 02000000:00000001:0.0:1787695764.790738:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695764.790745:0:42821:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695764.790748:0:42821:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695764.790750:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:1.0:1787695764.790757:0:42821:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000390bc45a] to pc [ptlrpcd_00_01+1] req@ffff8b1446667480 x1874531139519488/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1787695764.790759:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617baa8. 00000400:00000010:0.0:1787695764.790770:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96a18. 00000100:00000001:1.0:1787695764.790773:0:42821:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695764.790777:0:42821:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:0.0:1787695764.790780:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874531139519360, portal 10 00000100:00000040:1.0:1787695764.790782:0:42821:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1787695764.790785:0:2403:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695764.790788:0:2403:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929555712 : -128555779995904 : ffff8b1446664700) 00000100:00000040:1.0:1787695764.790788:0:42821:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1787695764.790795:0:42821:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1787695764.790800:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446664700 x1874531139519360/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695764.790803:0:42821:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 1 (3662) 00000100:00000001:0.0:1787695764.790811:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695764.790813:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695764.790815:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96000. 00000100:00000200:0.0:1787695764.790819:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874531139519360, offset 0 00000100:00000001:1.0:1787695764.790820:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695764.790824:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1787695764.790827:0:2403:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787695764.790828:0:2404:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000400:00000200:0.0:1787695764.790837:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:1.0:1787695764.790837:0:2404:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446667480 x1874531139519488/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695764.790850:0:2404:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695764.790852:0:2404:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695764.790854:0:2403:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695764.790859:0:2403:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695764.790866:0:2403:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 02000000:00000001:1.0:1787695764.790866:0:2404:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695764.790869:0:2404:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695764.790873:0:2404:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695764.790881:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00100000:1.0:1787695764.790881:0:2404:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446667480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:608c526c-eb6e-4a54-80a5-8969828241eb:2404:1874531139519488:192.168.201.140@tcp:400:kworker.0 00000400:00000200:0.0:1787695764.790886:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:1.0:1787695764.790892:0:2404:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:0.0:1787695764.790893:0:2403:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16868:16868:256:4294967295] 192.168.201.140@tcp LPNI seq info [16867:16867:8:4294967295] 02000000:00000001:1.0:1787695764.790898:0:2404:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:0.0:1787695764.790899:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:1.0:1787695764.790902:0:2404:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695764.790907:0:2404:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695764.790912:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b144667d600. 00000400:00000200:0.0:1787695764.790913:0:2403:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 02000000:00000001:1.0:1787695764.790918:0:2404:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1787695764.790924:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695764.790929:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000800:00000010:0.0:1787695764.790930:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da800. 00000400:00000010:1.0:1787695764.790935:0:2404:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702c08. 00000800:00000200:0.0:1787695764.790940:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (5) 00000400:00000010:1.0:1787695764.790944:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000800:00000200:0.0:1787695764.790949:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.140@tcp ip 192.168.201.140:988 00000100:00000200:1.0:1787695764.790951:0:2404:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874531139519488, portal 10 00000800:00000200:0.0:1787695764.790954:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8b14460da800 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1787695764.790956:0:2404:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695764.790960:0:2404:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929567360 : -128555779984256 : ffff8b1446667480) 00000100:00000001:0.0:1787695764.790969:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695764.790969:0:2404:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446667480 x1874531139519488/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695764.790976:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:0.0:1787695764.790984:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695764.790990:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298325253 00000100:00000001:1.0:1787695764.790990:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787695764.790996:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298332473 00000100:00000040:1.0:1787695764.790998:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:0.0:1787695764.791000:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298325253 00000400:00000010:1.0:1787695764.791004:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f465d8. 00000100:00000001:0.0:1787695764.791005:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298332473 00000100:00000001:0.0:1787695764.791009:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1787695764.791011:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874531139519488, offset 0 00000100:00000001:0.0:1787695764.791012:0:2403:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695764.791019:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695764.791023:0:2404:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:0.0:1787695764.791024:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0F:1787695764.791025:0:2398:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:1.0:1787695764.791032:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695764.791033:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695764.791037:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=61 : 61 : 3d) 00000800:00000010:3.0:1787695764.791038:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da800. 00000100:00000001:0.0:1787695764.791041:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1787695764.791042:0:2404:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:0.0:1787695764.791044:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1787695764.791048:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1787695764.791049:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695764.791050:0:2404:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1787695764.791055:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695764.791060:0:2404:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:3.0:1787695764.791061:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: OK 00000400:00000200:1.0:1787695764.791069:0:2404:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1787695764.791070:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96000 00000400:00000200:1.0:1787695764.791074:0:2404:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000010:3.0:1787695764.791076:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96000. 00000400:00000200:1.0:1787695764.791083:0:2404:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16869:16869:255:4294967295] 192.168.201.140@tcp LPNI seq info [16868:16868:7:4294967295] 00000100:00000001:3.0:1787695764.791083:0:2398:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:1.0:1787695764.791092:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000200:3.0:1787695764.791093:0:2398:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b1446664700 x1874531139519360/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695764.791111:0:2404:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:3.0:1787695764.791113:0:2398:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695764.791120:0:2398:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446664700 x1874531139519360/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695849 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1787695764.791132:0:2404:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695764.791136:0:2398:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1787695764.791139:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1445678800. 00000100:00000001:3.0:1787695764.791140:0:2398:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000200:1.0:1787695764.791147:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (5) 00000800:00000200:1.0:1787695764.791154:0:2404:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.140@tcp ip 192.168.201.140:988 00000800:00000200:1.0:1787695764.791157:0:2404:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8b1445678800 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1787695764.791168:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695764.791172:0:2404:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695764.791177:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695764.791180:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298325119 00000100:00000001:1.0:1787695764.791184:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298332473 00000100:00000001:1.0:1787695764.791187:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298325119 00000100:00000001:1.0:1787695764.791190:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298332473 00000800:00000200:3.0:1787695764.791190:0:2397:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1787695764.791195:0:2397:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1445678800. 00000100:00000001:1.0:1787695764.791197:0:2404:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1787695764.791201:0:2397:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1787695764.791202:0:2404:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695764.791210:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1787695764.791210:0:2397:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: OK 00000400:00000200:3.0:1787695764.791214:0:2397:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f465d8 00000100:00000001:1.0:1787695764.791216:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1787695764.791218:0:2397:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f465d8. 00000100:00000001:1.0:1787695764.791223:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695764.791223:0:2397:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787695764.791228:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=51 : 51 : 33) 00000100:00000200:3.0:1787695764.791231:0:2397:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b1446667480 x1874531139519488/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695849 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695764.791234:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695764.791237:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695764.791240:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695764.791247:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695764.791248:0:2397:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695764.791254:0:2397:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446667480 x1874531139519488/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695849 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695764.791262:0:2397:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695764.791264:0:2397:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695764.854441:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695764.854454:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695764.854457:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695764.854461:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695764.854464:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695764.854491:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695764.854494:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695765.046425:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695765.046431:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695765.046437:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695765.046438:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695765.046442:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695765.046443:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695765.046446:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695765.046447:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695765.046450:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695765.046452:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695765.046454:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695765.046463:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695765.046540:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695765.046544:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695765.046547:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787695765.814507:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695765.814516:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787695765.814534:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695765.814536:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1787695765.814538:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787695765.814541:0:38:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1787695765.814547:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446605500. 00000020:00000040:3.0:1787695765.814552:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=21 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000100:00000001:3.0:1787695765.814555:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695765.814558:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695765.814562:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695765.814570:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b1458095d00. 00000100:00000001:3.0:1787695765.814575:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695765.814584:0:38:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID req@ffff8b1446605500 x1874531139519616/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695765.814593:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695765.814596:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695765.814601:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007993d8e8] to pc [ptlrpcd_01_00+0] req@ffff8b1446605500 x1874531139519616/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695765.814623:0:38:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695765.814625:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695765.814628:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787695765.814630:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695765.814632:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787695765.814633:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695765.814635:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1787695765.814637:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787695765.814644:0:38:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1787695765.814648:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446605c00. 00000020:00000040:3.0:1787695765.814650:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=22 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000001:3.0:1787695765.814652:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695765.814653:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695765.814655:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695765.814657:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b1458095c00. 00000100:00000001:3.0:1787695765.814660:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0F:1787695765.814662:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1787695765.814664:0:38:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID req@ffff8b1446605c00 x1874531139519744/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695765.814670:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695765.814673:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695765.814677:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1787695765.814677:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000201ad08e] to pc [ptlrpcd_01_01+1] req@ffff8b1446605c00 x1874531139519744/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695765.814682:0:2405:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787695765.814688:0:38:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695765.814691:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1787695765.814695:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446605500 x1874531139519616/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695765.814695:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695765.814700:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787695765.814704:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695765.814710:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3663) 00000100:00000001:2.0:1787695765.814713:0:2405:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695765.814717:0:2405:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695765.814720:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787695765.814722:0:2405:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787695765.814724:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695765.814727:0:2406:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787695765.814728:0:2405:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1787695765.814732:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446605c00 x1874531139519744/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695765.814734:0:2405:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695765.814742:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:2.0:1787695765.814745:0:2405:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446605500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2405:1874531139519616:192.168.201.140@tcp:400:kworker.0 00000100:00000001:3.0:1787695765.814745:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695765.814749:0:2406:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787695765.814752:0:2406:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695765.814755:0:2405:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787695765.814760:0:2406:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695765.814762:0:2405:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:3.0:1787695765.814764:0:2406:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446605c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:608c526c-eb6e-4a54-80a5-8969828241eb:2406:1874531139519744:192.168.201.140@tcp:400:kworker.0 02000000:00000001:2.0:1787695765.814767:0:2405:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695765.814768:0:2406:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695765.814772:0:2405:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787695765.814772:0:2406:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787695765.814774:0:2406:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695765.814777:0:2406:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695765.814779:0:2405:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b14484ac200. 02000000:00000010:3.0:1787695765.814780:0:2406:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b1460531400. 02000000:00000001:2.0:1787695765.814783:0:2405:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695765.814784:0:2406:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695765.814794:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695765.814795:0:2406:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695765.814801:0:2405:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752b688. 00000400:00000010:3.0:1787695765.814802:0:2406:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14456d1370. 00000400:00000010:2.0:1787695765.814811:0:2405:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000400:00000010:3.0:1787695765.814812:0:2406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b145078a550. 00000100:00000200:2.0:1787695765.814821:0:2405:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874531139519616, portal 10 00000100:00000001:2.0:1787695765.814824:0:2405:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000200:3.0:1787695765.814825:0:2406:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874531139519744, portal 10 00000100:00000001:2.0:1787695765.814826:0:2405:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929166080 : -128555780385536 : ffff8b1446605500) 00000100:00000001:3.0:1787695765.814830:0:2406:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787695765.814833:0:2406:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929167872 : -128555780383744 : ffff8b1446605c00) 00000100:00000040:2.0:1787695765.814837:0:2405:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446605500 x1874531139519616/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695850 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695765.814841:0:2406:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446605c00 x1874531139519744/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695850 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695765.814859:0:2405:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787695765.814860:0:2406:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695765.814866:0:2405:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695765.814866:0:2406:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695765.814872:0:2405:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8198. 00000400:00000010:3.0:1787695765.814872:0:2406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b145078a5d8. 00000100:00000200:3.0:1787695765.814878:0:2406:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874531139519744, offset 0 00000400:00000200:3.0:1787695765.814883:0:2406:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000200:2.0:1787695765.814885:0:2405:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874531139519616, offset 0 00000400:00000200:3.0:1787695765.814891:0:2406:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695765.814895:0:2405:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:3.0:1787695765.814901:0:2406:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:3.0:1787695765.814906:0:2406:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1787695765.814909:0:2406:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:3.0:1787695765.814912:0:2406:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1787695765.814915:0:2406:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:3.0:1787695765.814918:0:2406:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16870:16870:256:4294967295] 192.168.201.140@tcp LPNI seq info [16869:16869:8:4294967295] 00000400:00000200:3.0:1787695765.814921:0:2406:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695765.814928:0:2406:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:3.0:1787695765.814933:0:2406:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695765.814934:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695765.814940:0:2405:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000800:00000010:3.0:1787695765.814942:0:2406:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1458095600. 00000400:00000200:2.0:1787695765.814943:0:2405:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695765.814948:0:2405:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000800:00000200:3.0:1787695765.814952:0:2406:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (5) 00000400:00000200:2.0:1787695765.814959:0:2405:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000800:00000200:3.0:1787695765.814963:0:2406:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.140@tcp ip 192.168.201.140:988 00000400:00000200:2.0:1787695765.814965:0:2405:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000800:00000200:3.0:1787695765.814969:0:2406:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8b1458095600 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1787695765.814970:0:2405:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16871:16871:255:4294967295] 192.168.201.140@tcp LPNI seq info [16870:16870:7:4294967295] 00000100:00000001:3.0:1787695765.814980:0:2406:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695765.814981:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695765.814985:0:2406:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:3.0:1787695765.814992:0:2406:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695765.814995:0:2405:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:3.0:1787695765.814997:0:2406:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298326143 00000800:00000200:2.0:1787695765.815002:0:2405:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695765.815006:0:2405:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5900. 00000100:00000001:3.0:1787695765.815010:0:2406:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298332473 00000800:00000200:2.0:1787695765.815013:0:2405:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (5) 00000100:00000001:3.0:1787695765.815017:0:2406:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298326143 00000800:00000200:2.0:1787695765.815020:0:2405:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.140@tcp ip 192.168.201.140:988 00000100:00000001:3.0:1787695765.815021:0:2406:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298332473 00000800:00000200:2.0:1787695765.815026:0:2405:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8b14517d5900 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1787695765.815027:0:2406:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695765.815032:0:2406:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695765.815034:0:2405:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695765.815038:0:2405:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000100:00000001:3.0:1787695765.815039:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695765.815043:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695765.815043:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695765.815047:0:2405:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298326277 00000100:00000001:2.0:1787695765.815050:0:2405:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298332473 00000100:00000001:3.0:1787695765.815051:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695765.815052:0:2405:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298326277 00000100:00000001:3.0:1787695765.815055:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=39 : 39 : 27) 00000100:00000001:2.0:1787695765.815056:0:2405:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298332473 00000100:00000001:3.0:1787695765.815059:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695765.815061:0:2405:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695765.815062:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695765.815065:0:2405:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695765.815067:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695765.815070:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695765.815073:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695765.815076:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695765.815080:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695765.815081:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=45 : 45 : 2d) 00000100:00000001:2.0:1787695765.815083:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695765.815084:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695765.815086:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695765.815088:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1787695765.815096:0:2398:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1787695765.815100:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1458095600. 00000400:00000200:3.0:1787695765.815106:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787695765.815110:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: OK 00000400:00000200:3.0:1787695765.815115:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b145078a5d8 00000400:00000010:3.0:1787695765.815117:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b145078a5d8. 00000100:00000001:3.0:1787695765.815120:0:2398:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1787695765.815123:0:2398:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b1446605c00 x1874531139519744/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695850 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695765.815130:0:2398:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695765.815132:0:2398:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446605c00 x1874531139519744/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695850 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695765.815142:0:2398:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695765.815143:0:2398:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000200:3.0:1787695765.815153:0:2398:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:3.0:1787695765.815155:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5900. 00000400:00000200:3.0:1787695765.815158:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787695765.815160:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: OK 00000400:00000200:3.0:1787695765.815162:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8198 00000400:00000010:3.0:1787695765.815163:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8198. 00000100:00000001:3.0:1787695765.815165:0:2398:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1787695765.815167:0:2398:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b1446605500 x1874531139519616/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695850 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695765.815172:0:2398:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695765.815174:0:2398:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446605500 x1874531139519616/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695850 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695765.815178:0:2398:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695765.815179:0:2398:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695765.878490:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695765.878495:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695765.878508:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695750/real 1787695750] req@ffff8b14465adf80 x1874531139518080/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695765 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695765.878530:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695765.878532:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695765.878535:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b2c0. 00000400:00000200:0.0:1787695765.878540:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695765.878542:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695765.878545:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695765.878548:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465adf80 x1874531139518080/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695765 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695765.878555:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465adf80 x1874531139518080/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695765 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695765.878560:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695765.878562:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695765.878564:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695765.878565:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695765.878568:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465adf80 x1874531139518080/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695765 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695765.878573:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695765.878575:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695765.878578:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695750/real 1787695750] req@ffff8b14465af800 x1874531139517952/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695765 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695765.878583:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695765.878583:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695765.878584:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b630. 00000400:00000200:0.0:1787695765.878586:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96aa0 00000400:00000010:0.0:1787695765.878587:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96aa0. 00000100:00000001:0.0:1787695765.878603:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695765.878606:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465af800 x1874531139517952/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695765 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695765.878611:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465af800 x1874531139517952/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695765 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695765.878615:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695765.878616:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695765.878617:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695765.878618:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695765.878620:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465af800 x1874531139517952/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695765 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695765.878624:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695765.878630:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695765.878631:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695765.878633:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695765.878635:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695765.878638:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695765.878640:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465adf80 x1874531139518080/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695765 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695765.878645:0:2402:0:(client.c:2142:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695765.878647:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695765.878648:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695765.878649:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695765.878651:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695765.878653:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695765.878655:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695765.878671:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695765.878673:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695765.878676:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465adf80 x1874531139518080/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695765 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695765.878691:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139518080:192.168.201.140@tcp:8:kworker.0 00000100:00080000:3.0:1787695765.878694:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695765.878696:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695765.878703:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787695765.878706:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1787695765.878708:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465af800 x1874531139517952/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695765 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695765.878709:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787695765.878714:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695765.878721:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000001:0.0:1787695765.878724:0:2402:0:(client.c:2142:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787695765.878725:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695765.878729:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695765.878731:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1787695765.878734:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695765.878734:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787695765.878738:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695765.878739:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695765.878743:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695765.878745:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695765.878751:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787695765.878752:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695765.878758:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3643 00000400:00000200:3.0:1787695765.878765:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695765.878770:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:0.0:1787695765.878774:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00001000:3.0:1787695765.878777:0:38:0:(import.c:2149:obd_at_measure()) add 15 to ffff8b1458080c18 time=71 v=10 (10 0 0 0) 00000100:00000001:0.0:1787695765.878779:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00001000:3.0:1787695765.878787:0:38:0:(import.c:2215:obd_at_measure()) AT ffff8b1458080c18 change: old=10 new=15 delta=5 (val=15) hist 15 0 0 0 00000100:00000040:0.0:1787695765.878790:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465af800 x1874531139517952/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695765 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695765.878792:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 10s 00000100:00000040:3.0:1787695765.878801:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00100000:0.0:1787695765.878805:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139517952:192.168.201.140@tcp:8:kworker.0 00000100:00000001:3.0:1787695765.878807:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695765.878809:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695765.878814:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695765.878814:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695765.878819:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000040:0.0:1787695765.878820:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465adf80 x1874531139518080/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695765 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695765.878826:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787695765.878828:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000020:00000040:3.0:1787695765.878830:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:3.0:1787695765.878835:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695765.878838:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:3.0:1787695765.878840:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:3.0:1787695765.878842:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:3.0:1787695765.878845:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695765.878847:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695765.878848:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695765.878850:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695765.878852:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:3.0:1787695765.878854:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695765.878856:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695765.878857:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787695765.878858:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:0.0:1787695765.878862:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545400. 02000000:00000001:3.0:1787695765.878865:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695765.878869:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:3.0:1787695765.878869:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695765.878873:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:3.0:1787695765.878873:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000020:00000040:0.0:1787695765.878876:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000008:00100000:3.0:1787695765.878879:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787695765.878881:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695765.878885:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695765.878887:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445620c00. 00000100:00000001:3.0:1787695765.878891:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695765.878892:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695765.878894:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695765.878896:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465adf80. 00000100:00000010:3.0:1787695765.878897:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446607800. 00000100:00000001:0.0:1787695765.878899:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000020:00000040:3.0:1787695765.878901:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 00000100:00000001:0.0:1787695765.878902:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695765.878906:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1787695765.878907:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695765.878909:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695765.878911:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465af800 x1874531139517952/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695765 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695765.878913:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695765.878916:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695765.878919:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695765.878925:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000010:3.0:1787695765.878926:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f5000. 02000000:00000001:0.0:1787695765.878927:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695765.878930:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547400. 00000100:00000001:3.0:1787695765.878932:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695765.878933:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695765.878937:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695765.878938:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:0.0:1787695765.878940:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695765.878942:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445621400. 00000100:00100000:3.0:1787695765.878943:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446607800 x1874531139519872/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787695765.878946:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695765.878948:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695765.878950:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465af800. 00000100:00000001:0.0:1787695765.878954:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695765.878956:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695765.878957:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695765.878959:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695765.878961:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695765.878967:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000075726140] to pc [ptlrpcd_rcv+-1] req@ffff8b1446607800 x1874531139519872/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695765.878988:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695765.878992:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695765.878996:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1787695765.878999:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787695765.879003:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695765.879006:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787695765.879009:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695765.879014:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3663) 00000100:00080000:3.0:1787695765.879019:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695765.879022:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000001:1.0:1787695765.879024:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787695765.879024:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695765.879027:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695765.879029:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1787695765.879032:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1787695765.879034:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446607800 x1874531139519872/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695765.879035:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695765.879038:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787695765.879040:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695765.879042:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695765.879043:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695765.879044:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695765.879045:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695765.879047:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3643 02000000:00000001:1.0:1787695765.879049:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:3.0:1787695765.879051:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:1.0:1787695765.879052:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1787695765.879054:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 02000000:00000001:1.0:1787695765.879055:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1787695765.879057:0:38:0:(import.c:2149:obd_at_measure()) add 15 to ffff8b1458083418 time=46 v=10 (10 1 1 1) 00000100:00100000:1.0:1787695765.879058:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446607800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139519872:192.168.201.140@tcp:8:kworker.0 00000100:00001000:3.0:1787695765.879061:0:38:0:(import.c:2215:obd_at_measure()) AT ffff8b1458083418 change: old=10 new=15 delta=5 (val=15) hist 15 1 1 1 00000100:00000001:1.0:1787695765.879062:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1787695765.879064:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 10s 00000100:00000040:3.0:1787695765.879066:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695765.879067:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695765.879069:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695765.879069:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695765.879071:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695765.879072:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787695765.879073:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695765.879076:0:38:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000010:1.0:1787695765.879077:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546000. 00000020:00000040:3.0:1787695765.879077:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 02000000:00000001:1.0:1787695765.879079:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695765.879079:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695765.879081:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:3.0:1787695765.879083:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000400:00000200:1.0:1787695765.879084:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:3.0:1787695765.879085:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000400:00000010:1.0:1787695765.879088:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702370. 00000100:00000040:3.0:1787695765.879088:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695765.879090:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695765.879092:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000010:1.0:1787695765.879093:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46000. 00000100:00000001:3.0:1787695765.879094:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695765.879096:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:3.0:1787695765.879098:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000200:1.0:1787695765.879099:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139519872, portal 4 00000100:00000001:1.0:1787695765.879101:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695765.879103:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929175040 : -128555780376576 : ffff8b1446607800) 00000100:00000001:3.0:1787695765.879104:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695765.879106:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695765.879109:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1787695765.879110:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446607800 x1874531139519872/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695785 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695765.879112:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695765.879114:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695765.879116:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787695765.879118:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:3.0:1787695765.879118:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695765.879120:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695765.879121:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000010:3.0:1787695765.879124:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446605180. 00000020:00000040:3.0:1787695765.879126:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:3.0:1787695765.879129:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695765.879130:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695765.879132:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f463b8. 00000100:00000001:3.0:1787695765.879132:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695765.879134:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:1.0:1787695765.879136:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139519872, offset 0 02000000:00000001:3.0:1787695765.879136:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695765.879140:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f7400. 00000400:00000200:1.0:1787695765.879143:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695765.879144:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695765.879148:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446605180 x1874531139520000/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695765.879150:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695765.879156:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:3.0:1787695765.879156:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695765.879159:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695765.879162:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005e7e02df] to pc [ptlrpcd_rcv+-1] req@ffff8b1446605180 x1874531139520000/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695765.879167:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695765.879171:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:3.0:1787695765.879171:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695765.879173:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695765.879175:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00080000:3.0:1787695765.879177:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000400:00000200:1.0:1787695765.879178:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000040:3.0:1787695765.879180:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695765.879182:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16872:16872:256:4294967295] 192.168.201.140@tcp LPNI seq info [16871:16871:8:4294967295] 00000100:00000040:3.0:1787695765.879183:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695765.879185:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695765.879187:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695765.879192:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695765.879195:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3663) 00000400:00000200:1.0:1787695765.879199:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695765.879205:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695765.879211:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1445678000. 00000800:00000200:1.0:1787695765.879217:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (5) 00000800:00000200:1.0:1787695765.879225:0:2402:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.140@tcp ip 192.168.201.140:988 00000800:00000200:1.0:1787695765.879230:0:2402:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8b1445678000 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:1.0:1787695765.879242:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695765.879246:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695765.879250:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695765.879253:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298326043 00000100:00000001:1.0:1787695765.879256:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298332473 00000100:00000001:1.0:1787695765.879258:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298326043 00000100:00000001:1.0:1787695765.879261:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298332473 00000100:00000001:1.0:1787695765.879263:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695765.879265:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695765.879268:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695765.879270:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695765.879273:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695765.879275:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1787695765.879275:0:2397:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1787695765.879277:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695765.879278:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000800:00000010:3.0:1787695765.879279:0:2397:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1445678000. 00000100:00000001:1.0:1787695765.879280:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695765.879284:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446605180 x1874531139520000/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695765.879284:0:2397:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787695765.879290:0:2397:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: OK 00000400:00000200:3.0:1787695765.879295:0:2397:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f463b8 00000400:00000010:3.0:1787695765.879297:0:2397:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f463b8. 00000100:00000001:3.0:1787695765.879301:0:2397:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1787695765.879306:0:2397:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b1446607800 x1874531139519872/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695785 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695765.879310:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695765.879312:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695765.879314:0:2397:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1787695765.879315:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695765.879317:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1787695765.879317:0:2397:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446607800 x1874531139519872/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695785 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695765.879319:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695765.879323:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446605180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139520000:192.168.201.140@tcp:8:kworker.0 00000100:00000001:3.0:1787695765.879325:0:2397:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695765.879326:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787695765.879327:0:2397:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1787695765.879328:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695765.879330:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695765.879332:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695765.879335:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695765.879337:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695765.879341:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695765.879345:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702058. 00000400:00000010:1.0:1787695765.879349:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695765.879352:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139520000, portal 4 00000100:00000001:1.0:1787695765.879354:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695765.879355:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929165184 : -128555780386432 : ffff8b1446605180) 00000100:00000040:1.0:1787695765.879361:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446605180 x1874531139520000/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695785 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695765.879396:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695765.879398:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695765.879401:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46cc0. 00000100:00000200:1.0:1787695765.879403:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139520000, offset 0 00000400:00000200:1.0:1787695765.879406:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695765.879409:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695765.879412:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695765.879414:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695765.879416:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695765.879417:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695765.879419:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695765.879425:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16873:16873:256:4294967295] 192.168.201.140@tcp LPNI seq info [16872:16872:8:4294967295] 00000400:00000200:1.0:1787695765.879429:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695765.879434:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695765.879437:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695765.879439:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1445678300. 00000800:00000200:1.0:1787695765.879442:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (5) 00000800:00000200:1.0:1787695765.879446:0:2402:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.140@tcp ip 192.168.201.140:988 00000800:00000200:1.0:1787695765.879448:0:2402:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8b1445678300 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:1.0:1787695765.879457:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695765.879472:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695765.879481:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695765.879484:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298326372 00000100:00000001:1.0:1787695765.879487:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298332473 00000100:00000001:1.0:1787695765.879489:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298326372 00000100:00000001:1.0:1787695765.879491:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298332473 00000100:00000001:1.0:1787695765.879493:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695765.879496:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1787695765.879498:0:2398:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1787695765.879499:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695765.879501:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:3.0:1787695765.879501:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1445678300. 00000100:00000001:1.0:1787695765.879505:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:3.0:1787695765.879505:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1787695765.879507:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695765.879509:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1787695765.879510:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: OK 00000100:00000001:1.0:1787695765.879511:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695765.879512:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1787695765.879513:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46cc0 00000100:00000001:1.0:1787695765.879515:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1787695765.879515:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46cc0. 00000100:00000001:3.0:1787695765.879518:0:2398:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1787695765.879523:0:2398:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b1446605180 x1874531139520000/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695785 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695765.879532:0:2398:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695765.879535:0:2398:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446605180 x1874531139520000/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695785 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695765.879542:0:2398:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695765.879544:0:2398:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1787695766.070501:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695766.070509:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695766.070515:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695766.070517:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695766.070520:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695766.070521:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695766.070523:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695766.070548:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695766.070550:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695766.070551:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695766.070554:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695766.070555:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695766.070621:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695766.070624:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695766.070627:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787695766.902435:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695766.902466:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695766.902468:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695766.902472:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695766.902474:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695766.902478:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695766.902480:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695767.094486:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695767.094492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695767.094497:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695767.094498:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695767.094501:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695767.094503:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695767.094505:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695767.094510:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695767.094512:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695767.094513:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695767.094515:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695767.094517:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695767.094519:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695767.094520:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695767.094523:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695767.094523:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695767.094524:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695767.094525:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695767.094526:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695767.094527:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695767.094528:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695767.094529:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695767.094530:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695767.094531:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695767.094534:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695767.094535:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695767.094536:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695767.094554:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695767.094555:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695767.094556:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695767.094557:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695767.094558:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695767.094559:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695767.094561:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695767.094563:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695767.094564:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695767.094565:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695767.094566:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695767.094567:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695767.094568:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695767.094569:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695767.094570:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695767.094571:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695767.094572:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695767.094623:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695767.094626:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695767.094629:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000001:2.0:1787695767.610205:0:52445:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1787695767.610217:0:52445:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695767.926511:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695767.926532:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695767.926534:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695767.926540:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695767.926542:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695767.926547:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695767.926550:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695768.118536:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695768.118544:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695768.118550:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695768.118551:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695768.118555:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695768.118556:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695768.118559:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695768.118560:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695768.118562:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695768.118563:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695768.118566:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695768.118567:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695768.118630:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695768.118653:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695768.118656:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787695768.950450:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695768.950463:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695768.950466:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695768.950488:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695768.950490:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695768.950494:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695768.950496:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695769.142495:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695769.142502:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695769.142508:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695769.142509:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695769.142513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695769.142515:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695769.142519:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695769.142520:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695769.142523:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695769.142525:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695769.142528:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695769.142529:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695769.142595:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695769.142599:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695769.142601:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787695769.974506:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695769.974518:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695769.974520:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695769.974540:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695769.974542:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695769.974546:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695769.974548:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695770.166479:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695770.166486:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695770.166493:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695770.166494:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695770.166496:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695770.166497:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695770.166500:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695770.166501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695770.166503:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695770.166504:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695770.166506:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695770.166506:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695770.166552:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695770.166556:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695770.166559:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.0:1787695770.294507:0:2394:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (5) 00000800:00000200:3.0:1787695770.294514:0:2394:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.140@tcp ip 192.168.201.140:988 00000800:00000200:3.0:1787695770.294519:0:2394:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8b1458095500 type 192, nob 24 niov 1 nkiov 0 00000800:00000200:2.0:1787695770.294913:0:2397:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1787695770.998450:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695770.998464:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1787695770.998466:0:39415:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695770.998467:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695770.998475:0:39415:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000001:0.0:1787695770.998476:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787695770.998479:0:39415:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695770.998481:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1787695770.998485:0:39415:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1787695770.998488:0:39415:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695770.998489:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695770.998492:0:39415:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1787695770.998494:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695770.998501:0:39415:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7b80. 00000020:00000040:3.0:1787695770.998507:0:39415:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=22 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000100:00000001:3.0:1787695770.998511:0:39415:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695770.998514:0:39415:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695770.998519:0:39415:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695770.998527:0:39415:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b1458095600. 00000100:00000001:3.0:1787695770.998532:0:39415:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695770.998544:0:39415:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID req@ffff8b14466f7b80 x1874531139520128/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695770.998552:0:39415:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695770.998556:0:39415:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695770.998562:0:39415:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b87e1da] to pc [ptlrpcd_01_00+0] req@ffff8b14466f7b80 x1874531139520128/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695770.998580:0:39415:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695770.998582:0:39415:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695770.998585:0:39415:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787695770.998594:0:39415:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695770.998597:0:39415:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787695770.998599:0:39415:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695770.998601:0:39415:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:3.0:1787695770.998603:0:39415:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787695770.998605:0:39415:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1787695770.998608:0:39415:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6a00. 00000020:00000040:3.0:1787695770.998610:0:39415:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=23 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000001:2.0:1787695770.998611:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695770.998613:0:39415:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695770.998614:0:39415:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695770.998616:0:39415:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695770.998621:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000010:3.0:1787695770.998623:0:39415:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b1461888000. 00000100:00000001:2.0:1787695770.998624:0:2405:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787695770.998629:0:39415:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695770.998633:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7b80 x1874531139520128/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695770.998638:0:39415:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID req@ffff8b14466f6a00 x1874531139520256/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695770.998645:0:39415:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695770.998649:0:39415:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695770.998656:0:2405:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1787695770.998657:0:39415:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002ad47e8f] to pc [ptlrpcd_01_01+1] req@ffff8b14466f6a00 x1874531139520256/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695770.998659:0:2405:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695770.998665:0:2405:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787695770.998668:0:39415:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695770.998669:0:2405:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1787695770.998672:0:39415:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1787695770.998673:0:2405:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695770.998680:0:39415:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00100000:2.0:1787695770.998682:0:2405:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2405:1874531139520128:192.168.201.140@tcp:400:kworker.0 00000100:00000040:3.0:1787695770.998684:0:39415:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787695770.998690:0:2405:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695770.998695:0:2405:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695770.998697:0:2405:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695770.998699:0:2405:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695770.998702:0:2405:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b14484ac400. 02000000:00000001:2.0:1787695770.998704:0:2405:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695770.998710:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695770.998714:0:2405:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695770.998720:0:2405:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695770.998725:0:2405:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874531139520128, portal 10 00000100:00000001:2.0:1787695770.998727:0:2405:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695770.998729:0:2405:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930158976 : -128555779392640 : ffff8b14466f7b80) 00000100:00000040:2.0:1787695770.998735:0:2405:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7b80 x1874531139520128/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695855 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695770.998741:0:2405:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695770.998743:0:2405:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695770.998745:0:2405:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8b28. 00000100:00000200:2.0:1787695770.998752:0:2405:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874531139520128, offset 0 00000100:00000040:3.0:1787695770.998756:0:39415:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:2.0:1787695770.998760:0:2405:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695770.998763:0:39415:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3668) 00000400:00000200:2.0:1787695770.998769:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695770.998779:0:2405:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695770.998785:0:2405:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1787695770.998786:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695770.998789:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1787695770.998790:0:2405:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:3.0:1787695770.998792:0:2406:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1787695770.998798:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6a00 x1874531139520256/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787695770.998803:0:2405:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695770.998808:0:2405:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695770.998812:0:2405:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16874:16874:256:4294967295] 192.168.201.140@tcp LPNI seq info [16873:16873:8:4294967295] 00000400:00000200:2.0:1787695770.998815:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695770.998819:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695770.998820:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695770.998824:0:2406:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787695770.998826:0:2406:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695770.998828:0:2406:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695770.998832:0:2405:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00100000:3.0:1787695770.998836:0:2406:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:608c526c-eb6e-4a54-80a5-8969828241eb:2406:1874531139520256:192.168.201.140@tcp:400:kworker.0 00000800:00000200:2.0:1787695770.998842:0:2405:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695770.998843:0:2406:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000800:00000010:2.0:1787695770.998848:0:2405:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5300. 02000000:00000001:3.0:1787695770.998848:0:2406:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787695770.998850:0:2406:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695770.998853:0:2406:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1787695770.998857:0:2406:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b1460530a00. 00000800:00000200:2.0:1787695770.998858:0:2405:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (5) 00000800:00000200:2.0:1787695770.998864:0:2405:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.140@tcp ip 192.168.201.140:988 00000800:00000200:2.0:1787695770.998868:0:2405:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8b14517d5300 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1787695770.998880:0:2405:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695770.998883:0:2405:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:2.0:1787695770.998886:0:2405:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695770.998889:0:2405:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298331461 00000100:00000001:2.0:1787695770.998892:0:2405:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298332473 00000100:00000001:2.0:1787695770.998894:0:2405:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298331461 00000100:00000001:2.0:1787695770.998895:0:2405:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298332473 00000100:00000001:2.0:1787695770.998897:0:2405:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695770.998899:0:2405:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695770.998903:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695770.998905:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695770.998908:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695770.998909:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=40 : 40 : 28) 00000100:00000001:2.0:1787695770.998911:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695770.998912:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695770.998914:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695770.998916:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1787695770.998961:0:2398:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1787695770.998965:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5300. 02000000:00000001:3.0:1787695770.998966:0:2406:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695770.998970:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:3.0:1787695770.998974:0:2406:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695770.998979:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: OK 00000400:00000010:3.0:1787695770.998979:0:2406:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14456d1688. 00000400:00000010:3.0:1787695770.998986:0:2406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b145078a5d8. 00000400:00000200:2.0:1787695770.998989:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8b28 00000400:00000010:2.0:1787695770.998993:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8b28. 00000100:00000001:2.0:1787695770.998997:0:2398:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1787695770.998998:0:2406:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874531139520256, portal 10 00000100:00000200:2.0:1787695770.999003:0:2398:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b14466f7b80 x1874531139520128/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695855 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695770.999003:0:2406:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787695770.999007:0:2406:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930154496 : -128555779397120 : ffff8b14466f6a00) 00000100:00000001:2.0:1787695770.999017:0:2398:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695770.999020:0:2406:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6a00 x1874531139520256/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695855 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695770.999024:0:2398:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7b80 x1874531139520128/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695855 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695770.999044:0:2398:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695770.999046:0:2398:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787695770.999085:0:2406:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1787695770.999088:0:2406:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:3.0:1787695770.999090:0:2406:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b145078a4c8. 00000100:00000200:3.0:1787695770.999092:0:2406:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874531139520256, offset 0 00000400:00000200:3.0:1787695770.999095:0:2406:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:3.0:1787695770.999100:0:2406:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695770.999103:0:2406:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:3.0:1787695770.999105:0:2406:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1787695770.999112:0:2406:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:3.0:1787695770.999114:0:2406:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1787695770.999116:0:2406:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:3.0:1787695770.999118:0:2406:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16875:16875:256:4294967295] 192.168.201.140@tcp LPNI seq info [16874:16874:8:4294967295] 00000400:00000200:3.0:1787695770.999121:0:2406:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695770.999127:0:2406:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:3.0:1787695770.999130:0:2406:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:3.0:1787695770.999133:0:2406:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461888900. 00000800:00000200:3.0:1787695770.999136:0:2406:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (5) 00000800:00000200:3.0:1787695770.999140:0:2406:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.140@tcp ip 192.168.201.140:988 00000800:00000200:3.0:1787695770.999142:0:2406:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8b1461888900 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1787695770.999153:0:2406:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695770.999156:0:2406:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:3.0:1787695770.999158:0:2406:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695770.999160:0:2406:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298331327 00000100:00000001:3.0:1787695770.999162:0:2406:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298332473 00000100:00000001:3.0:1787695770.999164:0:2406:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298331327 00000100:00000001:3.0:1787695770.999165:0:2406:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298332473 00000100:00000001:3.0:1787695770.999167:0:2406:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695770.999169:0:2406:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695770.999174:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695770.999176:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695770.999179:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695770.999181:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=34 : 34 : 22) 00000100:00000001:3.0:1787695770.999183:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695770.999184:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695770.999186:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695770.999188:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1787695770.999192:0:2397:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1787695770.999196:0:2397:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461888900. 00000400:00000200:2.0:1787695770.999200:0:2397:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1787695770.999206:0:2397:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: OK 00000400:00000200:2.0:1787695770.999210:0:2397:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b145078a4c8 00000400:00000010:2.0:1787695770.999212:0:2397:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b145078a4c8. 00000100:00000001:2.0:1787695770.999216:0:2397:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695770.999221:0:2397:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b14466f6a00 x1874531139520256/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695855 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695770.999230:0:2397:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695770.999234:0:2397:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6a00 x1874531139520256/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695855 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695770.999246:0:2397:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695770.999248:0:2397:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1787695771.190516:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695771.190527:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695771.190535:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695771.190536:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695771.190540:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695771.190541:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695771.190544:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695771.190545:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695771.190548:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695771.190548:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695771.190575:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695771.190576:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695771.190644:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695771.190649:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695771.190653:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000001:0.0:1787695771.463647:0:52503:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1787695771.463659:0:52503:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695771.589611:0:52523:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffa31781d17d08 00000080:00200008:3.0:1787695771.589617:0:52523:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1450dbb248), request_mask 2047, flags 0, foreign 0 00000080:00000001:3.0:1787695771.589623:0:52523:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:3.0:1787695771.589625:0:52523:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffa31781d17c80 00000080:00200000:3.0:1787695771.589626:0:52523:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1450dbb248),name=/ 00000080:00000010:3.0:1787695771.589630:0:52523:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff8b144667c600. 00800000:00000002:3.0:1787695771.589635:0:52523:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff8b1444613300 2 00000080:00000001:3.0:1787695771.589638:0:52523:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:3.0:1787695771.589642:0:52523:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:3.0:1787695771.589644:0:52523:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffa31781d17ba0 00800000:00000002:3.0:1787695771.589646:0:52523:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:3.0:1787695771.589648:0:52523:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:3.0:1787695771.589651:0:52523:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:3.0:1787695771.589652:0:52523:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:3.0:1787695771.589654:0:52523:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:3.0:1787695771.589656:0:52523:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:3.0:1787695771.589658:0:52523:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:3.0:1787695771.589659:0:52523:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:3.0:1787695771.589661:0:52523:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:3.0:1787695771.589663:0:52523:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:3.0:1787695771.589668:0:52523:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:3.0:1787695771.589669:0:52523:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffa31781d17a70 00000002:00010000:3.0:1787695771.589671:0:52523:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:3.0:1787695771.589673:0:52523:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:3.0:1787695771.589675:0:52523:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:3.0:1787695771.589678:0:52523:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:3.0:1787695771.589683:0:52523:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff8b1451e83800 count: 2 00010000:00000001:3.0:1787695771.589685:0:52523:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1787695771.589689:0:52523:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1787695771.589692:0:52523:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:3.0:1787695771.589696:0:52523:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b1458183800 lock: ffff8b14487aee00/0xae91b608b3bfabf6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x5a4cfdf27ef60225 expref: -99 pid: 51881 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 02000000:00000001:3.0:1787695771.589704:0:52523:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:3.0:1787695771.589713:0:52523:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695771.589715:0:52523:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b1458183800 lock: ffff8b14487aee00/0xae91b608b3bfabf6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x5a4cfdf27ef60225 expref: -99 pid: 51881 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000002:00000001:3.0:1787695771.589720:0:52523:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:3.0:1787695771.589722:0:52523:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:3.0:1787695771.589723:0:52523:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:3.0:1787695771.589724:0:52523:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:3.0:1787695771.589726:0:52523:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:3.0:1787695771.589727:0:52523:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00000001:3.0:1787695771.589729:0:52523:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:3.0:1787695771.589731:0:52523:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff8b1444613300 2 00000080:00000010:3.0:1787695771.589733:0:52523:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff8b144667c600. 00000080:00000001:3.0:1787695771.589736:0:52523:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:3.0:1787695771.589737:0:52523:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:3.0:1787695771.589739:0:52523:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1450dbb248) 00000080:00010000:3.0:1787695771.589741:0:52523:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1450dbb248) for lock 0xae91b608b3bfabf6 00800000:00000001:3.0:1787695771.589744:0:52523:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:3.0:1787695771.589745:0:52523:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:3.0:1787695771.589747:0:52523:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1787695771.589748:0:52523:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695771.589750:0:52523:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff8b14487aee00 refcount=3 00000020:00000001:3.0:1787695771.589751:0:52523:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517964463616 : -128555745088000 : ffff8b14487aee00) 00010000:00000001:3.0:1787695771.589753:0:52523:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615517964463616 : -128555745088000 : ffff8b14487aee00) 00010000:00000001:3.0:1787695771.589755:0:52523:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1787695771.589756:0:52523:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:3.0:1787695771.589757:0:52523:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:3.0:1787695771.589758:0:52523:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:3.0:1787695771.589760:0:52523:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xae91b608b3bfabf6 from it ffffa31781d17cb8 00010000:00000001:3.0:1787695771.589761:0:52523:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1787695771.589762:0:52523:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695771.589763:0:52523:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff8b14487aee00 refcount=3 00000020:00000001:3.0:1787695771.589765:0:52523:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517964463616 : -128555745088000 : ffff8b14487aee00) 00010000:00000001:3.0:1787695771.589766:0:52523:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615517964463616 : -128555745088000 : ffff8b14487aee00) 00010000:00000001:3.0:1787695771.589770:0:52523:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:3.0:1787695771.589773:0:52523:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b1458183800 lock: ffff8b14487aee00/0xae91b608b3bfabf6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x5a4cfdf27ef60225 expref: -99 pid: 51881 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00000001:3.0:1787695771.589778:0:52523:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1787695771.589778:0:52523:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1787695771.589779:0:52523:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:3.0:1787695771.589781:0:52523:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:3.0:1787695771.589783:0:52523:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b1458183800 lock: ffff8b14487aee00/0xae91b608b3bfabf6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x5a4cfdf27ef60225 expref: -99 pid: 51881 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00000001:3.0:1787695771.589789:0:52523:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787695771.589791:0:52523:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1787695771.589793:0:52523:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1787695771.589796:0:52523:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1787695771.589797:0:52523:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1787695771.589798:0:52523:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787695771.589800:0:52523:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1787695771.589801:0:52523:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787695771.589803:0:52523:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787695771.589804:0:52523:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787695771.589805:0:52523:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:3.0:1787695771.589806:0:52523:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1787695771.589807:0:52523:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1787695771.589810:0:52523:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787695771.589811:0:52523:0:(client.c:2869:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:3.0:1787695771.589813:0:52523:0:(file.c:6249:ll_merge_md_attr()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695771.589816:0:52523:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffa31781d17d08 00000080:00000008:3.0:1787695771.589817:0:52523:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file /:[0x200000007:0x1:0x0](ffff8b1450dbb248), request_mask 2047, flags 0, foreign 0 00000080:00000001:3.0:1787695771.590205:0:52523:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1787695771.590211:0:52523:0:(llite_lib.c:4247:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695771.590228:0:52523:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffa31781d17d08 00000080:00200008:3.0:1787695771.590230:0:52523:0:(file.c:6296:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff8b1450dbb248), request_mask 0, flags 0, foreign 0 00000080:00000001:3.0:1787695771.590233:0:52523:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:3.0:1787695771.590234:0:52523:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffa31781d17c80 00000080:00200000:3.0:1787695771.590235:0:52523:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1450dbb248),name=/ 00000080:00000010:3.0:1787695771.590241:0:52523:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff8b144667c600. 00800000:00000002:3.0:1787695771.590243:0:52523:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff8b1444613300 2 00000080:00000001:3.0:1787695771.590245:0:52523:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:3.0:1787695771.590247:0:52523:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:3.0:1787695771.590248:0:52523:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffa31781d17ba0 00800000:00000002:3.0:1787695771.590249:0:52523:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:3.0:1787695771.590251:0:52523:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:3.0:1787695771.590252:0:52523:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:3.0:1787695771.590253:0:52523:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:3.0:1787695771.590254:0:52523:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:3.0:1787695771.590255:0:52523:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:3.0:1787695771.590256:0:52523:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:3.0:1787695771.590258:0:52523:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:3.0:1787695771.590259:0:52523:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:3.0:1787695771.590260:0:52523:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:3.0:1787695771.590263:0:52523:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:3.0:1787695771.590264:0:52523:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffa31781d17a70 00000002:00010000:3.0:1787695771.590265:0:52523:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:3.0:1787695771.590267:0:52523:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:3.0:1787695771.590268:0:52523:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:3.0:1787695771.590269:0:52523:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:3.0:1787695771.590271:0:52523:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff8b1451e83800 count: 2 00010000:00000001:3.0:1787695771.590272:0:52523:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1787695771.590274:0:52523:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1787695771.590276:0:52523:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:3.0:1787695771.590278:0:52523:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b1458183800 lock: ffff8b14487aee00/0xae91b608b3bfabf6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x5a4cfdf27ef60225 expref: -99 pid: 51881 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 02000000:00000001:3.0:1787695771.590283:0:52523:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:3.0:1787695771.590285:0:52523:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695771.590287:0:52523:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b1458183800 lock: ffff8b14487aee00/0xae91b608b3bfabf6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x5a4cfdf27ef60225 expref: -99 pid: 51881 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000002:00000001:3.0:1787695771.590291:0:52523:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:3.0:1787695771.590295:0:52523:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:3.0:1787695771.590296:0:52523:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:3.0:1787695771.590297:0:52523:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:3.0:1787695771.590299:0:52523:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:3.0:1787695771.590300:0:52523:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00000001:3.0:1787695771.590302:0:52523:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:3.0:1787695771.590303:0:52523:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff8b1444613300 2 00000080:00000010:3.0:1787695771.590305:0:52523:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff8b144667c600. 00000080:00000001:3.0:1787695771.590306:0:52523:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:3.0:1787695771.590307:0:52523:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:3.0:1787695771.590309:0:52523:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1450dbb248) 00000080:00010000:3.0:1787695771.590310:0:52523:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1450dbb248) for lock 0xae91b608b3bfabf6 00800000:00000001:3.0:1787695771.590313:0:52523:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:3.0:1787695771.590314:0:52523:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:3.0:1787695771.590315:0:52523:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1787695771.590316:0:52523:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695771.590317:0:52523:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff8b14487aee00 refcount=3 00000020:00000001:3.0:1787695771.590319:0:52523:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517964463616 : -128555745088000 : ffff8b14487aee00) 00010000:00000001:3.0:1787695771.590320:0:52523:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615517964463616 : -128555745088000 : ffff8b14487aee00) 00010000:00000001:3.0:1787695771.590322:0:52523:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1787695771.590323:0:52523:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:3.0:1787695771.590324:0:52523:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:3.0:1787695771.590325:0:52523:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:3.0:1787695771.590326:0:52523:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xae91b608b3bfabf6 from it ffffa31781d17cb8 00010000:00000001:3.0:1787695771.590328:0:52523:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1787695771.590328:0:52523:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695771.590329:0:52523:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff8b14487aee00 refcount=3 00000020:00000001:3.0:1787695771.590330:0:52523:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517964463616 : -128555745088000 : ffff8b14487aee00) 00010000:00000001:3.0:1787695771.590331:0:52523:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615517964463616 : -128555745088000 : ffff8b14487aee00) 00010000:00000001:3.0:1787695771.590333:0:52523:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:3.0:1787695771.590334:0:52523:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b1458183800 lock: ffff8b14487aee00/0xae91b608b3bfabf6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x5a4cfdf27ef60225 expref: -99 pid: 51881 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00000001:3.0:1787695771.590338:0:52523:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1787695771.590341:0:52523:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1787695771.590342:0:52523:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:3.0:1787695771.590343:0:52523:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:3.0:1787695771.590345:0:52523:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b1458183800 lock: ffff8b14487aee00/0xae91b608b3bfabf6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x5a4cfdf27ef60225 expref: -99 pid: 51881 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00000001:3.0:1787695771.590349:0:52523:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787695771.590350:0:52523:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1787695771.590351:0:52523:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1787695771.590353:0:52523:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1787695771.590353:0:52523:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1787695771.590354:0:52523:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787695771.590356:0:52523:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1787695771.590357:0:52523:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787695771.590358:0:52523:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787695771.590360:0:52523:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787695771.590389:0:52523:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:3.0:1787695771.590391:0:52523:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1787695771.590392:0:52523:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1787695771.590394:0:52523:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787695771.590396:0:52523:0:(client.c:2869:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:3.0:1787695771.590398:0:52523:0:(file.c:6249:ll_merge_md_attr()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695771.590400:0:52523:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffa31781d17d08 00000080:00000008:3.0:1787695771.590402:0:52523:0:(file.c:6481:ll_getattr_dentry()) COMPLETED file /:[0x200000007:0x1:0x0](ffff8b1450dbb248), request_mask 0, flags 0, foreign 0 00000080:00000001:3.0:1787695771.590454:0:52523:0:(file.c:6657:ll_inode_permission()) Process entered 00000080:00000001:3.0:1787695771.590455:0:52523:0:(file.c:6189:ll_inode_revalidate()) Process entered 00000020:00000001:3.0:1787695771.590456:0:52523:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffa31781d17b78 00000080:00200000:3.0:1787695771.590458:0:52523:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1450dbb248),name=/ 00000080:00000010:3.0:1787695771.590461:0:52523:0:(llite_lib.c:4058:ll_prep_md_op_data()) kmalloced '(op_data)': 392 at ffff8b144667c600. 00800000:00000002:3.0:1787695771.590463:0:52523:0:(lmv_obd.c:4002:lmv_stripe_object_get()) get ffff8b1444613300 2 00000080:00000001:3.0:1787695771.590464:0:52523:0:(namei.c:944:ll_intent_lock()) Process entered 00800000:00000001:3.0:1787695771.590466:0:52523:0:(lmv_intent.c:566:lmv_intent_lock()) Process entered 00000020:00000001:3.0:1787695771.590467:0:52523:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffa31781d17a98 00800000:00000002:3.0:1787695771.590468:0:52523:0:(lmv_intent.c:573:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:3.0:1787695771.590470:0:52523:0:(lmv_intent.c:447:lmv_intent_lookup()) Process entered 00800000:00000001:3.0:1787695771.590471:0:52523:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:3.0:1787695771.590472:0:52523:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:3.0:1787695771.590476:0:52523:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:3.0:1787695771.590478:0:52523:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:3.0:1787695771.590479:0:52523:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:3.0:1787695771.590480:0:52523:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:3.0:1787695771.590481:0:52523:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:3.0:1787695771.590483:0:52523:0:(lmv_intent.c:500:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:3.0:1787695771.590485:0:52523:0:(mdc_locks.c:1502:mdc_intent_lock()) Process entered 00000020:00000001:3.0:1787695771.590486:0:52523:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffa31781d17968 00000002:00010000:3.0:1787695771.590487:0:52523:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:3.0:1787695771.590489:0:52523:0:(mdc_locks.c:1406:mdc_revalidate_lock()) Process entered 00000002:00000001:3.0:1787695771.590490:0:52523:0:(mdc_locks.c:137:mdc_lock_match()) Process entered 00010000:00000001:3.0:1787695771.590491:0:52523:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:3.0:1787695771.590493:0:52523:0:(ldlm_resource.c:1656:ldlm_resource_getref()) getref res: ffff8b1451e83800 count: 2 00010000:00000001:3.0:1787695771.590495:0:52523:0:(ldlm_resource.c:1595:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1787695771.590496:0:52523:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1787695771.590498:0:52523:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:3.0:1787695771.590500:0:52523:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff8b1458183800 lock: ffff8b14487aee00/0xae91b608b3bfabf6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0x5a4cfdf27ef60225 expref: -99 pid: 51881 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 02000000:00000001:3.0:1787695771.590505:0:52523:0:(sec.c:1025:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:3.0:1787695771.590507:0:52523:0:(sec.c:1041:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695771.590509:0:52523:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff8b1458183800 lock: ffff8b14487aee00/0xae91b608b3bfabf6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x5a4cfdf27ef60225 expref: -99 pid: 51881 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000002:00000001:3.0:1787695771.590513:0:52523:0:(mdc_locks.c:143:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:3.0:1787695771.590514:0:52523:0:(mdc_locks.c:1458:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:3.0:1787695771.590515:0:52523:0:(mdc_locks.c:1525:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:3.0:1787695771.590516:0:52523:0:(lmv_intent.c:521:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:3.0:1787695771.590517:0:52523:0:(lmv_intent.c:606:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:3.0:1787695771.590518:0:52523:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00000001:3.0:1787695771.590520:0:52523:0:(namei.c:990:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:3.0:1787695771.590521:0:52523:0:(lmv_obd.c:4047:lmv_stripe_object_put()) put ffff8b1444613300 2 00000080:00000010:3.0:1787695771.590523:0:52523:0:(llite_lib.c:4199:ll_finish_md_op_data()) kfreed 'op_data': 392 at ffff8b144667c600. 00000080:00000001:3.0:1787695771.590524:0:52523:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:3.0:1787695771.590528:0:52523:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:3.0:1787695771.590529:0:52523:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1450dbb248) 00000080:00010000:3.0:1787695771.590531:0:52523:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff8b1450dbb248) for lock 0xae91b608b3bfabf6 00800000:00000001:3.0:1787695771.590533:0:52523:0:(lmv_obd.c:4087:lmv_set_lock_data()) Process entered 00000002:00000001:3.0:1787695771.590534:0:52523:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:3.0:1787695771.590535:0:52523:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1787695771.590536:0:52523:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695771.590537:0:52523:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff8b14487aee00 refcount=3 00000020:00000001:3.0:1787695771.590538:0:52523:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517964463616 : -128555745088000 : ffff8b14487aee00) 00010000:00000001:3.0:1787695771.590540:0:52523:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615517964463616 : -128555745088000 : ffff8b14487aee00) 00010000:00000001:3.0:1787695771.590542:0:52523:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1787695771.590542:0:52523:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:3.0:1787695771.590543:0:52523:0:(mdc_locks.c:124:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:3.0:1787695771.590545:0:52523:0:(lmv_obd.c:4092:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:3.0:1787695771.590546:0:52523:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xae91b608b3bfabf6 from it ffffa31781d17bb0 00010000:00000001:3.0:1787695771.590547:0:52523:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1787695771.590548:0:52523:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695771.590549:0:52523:0:(lustre_handles.c:148:class_handle2object()) GET ldlm ffff8b14487aee00 refcount=3 00000020:00000001:3.0:1787695771.590550:0:52523:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517964463616 : -128555745088000 : ffff8b14487aee00) 00010000:00000001:3.0:1787695771.590551:0:52523:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446615517964463616 : -128555745088000 : ffff8b14487aee00) 00010000:00000001:3.0:1787695771.590552:0:52523:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:3.0:1787695771.590554:0:52523:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff8b1458183800 lock: ffff8b14487aee00/0xae91b608b3bfabf6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0x5a4cfdf27ef60225 expref: -99 pid: 51881 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00000001:3.0:1787695771.590558:0:52523:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1787695771.590558:0:52523:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1787695771.590559:0:52523:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:3.0:1787695771.590560:0:52523:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:3.0:1787695771.590562:0:52523:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff8b1458183800 lock: ffff8b14487aee00/0xae91b608b3bfabf6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0x5a4cfdf27ef60225 expref: -99 pid: 51881 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 0 00010000:00000001:3.0:1787695771.590566:0:52523:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1787695771.590567:0:52523:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:3.0:1787695771.590570:0:52523:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:3.0:1787695771.590572:0:52523:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1787695771.590573:0:52523:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1787695771.590574:0:52523:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787695771.590575:0:52523:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:3.0:1787695771.590576:0:52523:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787695771.590577:0:52523:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787695771.590579:0:52523:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1787695771.590580:0:52523:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:3.0:1787695771.590581:0:52523:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1787695771.590582:0:52523:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1787695771.590583:0:52523:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787695771.590584:0:52523:0:(client.c:2869:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:3.0:1787695771.590586:0:52523:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1450dbb248), inode mode 41ed mask 44 00000080:00000001:3.0:1787695771.590588:0:52523:0:(file.c:6711:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1787695771.590590:0:52523:0:(dir.c:3008:ll_dir_open()) Process entered 00000080:00000001:3.0:1787695771.590592:0:52523:0:(file.c:1086:ll_file_open()) Process entered 00000020:00000001:3.0:1787695771.590593:0:52523:0:(obd_sysfs.c:926:encode_fn_len()) Process filename at ffffa31781d17b78 00000080:00200008:3.0:1787695771.590595:0:52523:0:(file.c:1090:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff8b1450dbb248), flags 100000 00000080:00000010:3.0:1787695771.590598:0:52523:0:(file.c:70:ll_file_data_get()) slab-alloced 'lfd': 288 at ffff8b1442b9ea00. 00000080:00000001:3.0:1787695771.590602:0:52523:0:(file.c:1115:ll_file_open()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1787695771.590603:0:52523:0:(dir.c:3009:ll_dir_open()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1787695771.590654:0:52523:0:(dir.c:1999:ll_dir_ioctl()) Process entered 00000080:00200080:3.0:1787695771.590656:0:52523:0:(dir.c:2001:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff8b1450dbb248) cmd=c00866a4 arg=7ffdbb379a00 00000080:00000001:3.0:1787695771.590662:0:52523:0:(llite_lib.c:3554:ll_iocontrol()) Process entered 00000020:00000001:3.0:1787695771.590664:0:52523:0:(class_obd.c:259:obd_ioctl_getdata()) Process entered 00000020:00000010:3.0:1787695771.590667:0:52523:0:(class_obd.c:286:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff8b14464f7000. 00000020:00000001:3.0:1787695771.590670:0:52523:0:(class_obd.c:321:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1787695771.590672:0:52523:0:(obd_class.h:1200:obd_iocontrol()) Process entered 00800000:00000001:3.0:1787695771.590674:0:52523:0:(lmv_obd.c:932:lmv_iocontrol()) Process entered 00800000:00000080:3.0:1787695771.590676:0:52523:0:(lmv_obd.c:934:lmv_iocontrol()) lustre-clilmv-ffff8b1458183800: cmd=c00866a4 len=592 karg=ffff8b14464f7000 uarg=0000000000000000 00800000:00000001:3.0:1787695771.590680:0:52523:0:(obd_class.h:1074:obd_statfs()) Process entered 00800000:00000004:3.0:1787695771.590682:0:52523:0:(obd_class.h:1090:obd_statfs()) lustre-MDT0000-mdc-ffff8b1458183800: age 3600, max_age 3663 00000002:00000001:3.0:1787695771.590685:0:52523:0:(mdc_request.c:1667:mdc_statfs()) Process entered 00000020:00000040:3.0:1787695771.590687:0:52523:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000100:00000010:3.0:1787695771.590691:0:52523:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6300. 00000020:00000040:3.0:1787695771.590693:0:52523:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000100:00000001:3.0:1787695771.590702:0:52523:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695771.590704:0:52523:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695771.590706:0:52523:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695771.590711:0:52523:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b1461888400. 00000100:00000001:3.0:1787695771.590716:0:52523:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695771.590719:0:52523:0:(client.c:3352:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1787695771.590720:0:52523:0:(client.c:1169:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1787695771.590724:0:52523:0:(client.c:1171:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff8b1460d9b6c0. 00000100:00000001:3.0:1787695771.590726:0:52523:0:(client.c:1186:ptlrpc_prep_set()) Process leaving (rc=18446615518373328576 : -128555336223040 : ffff8b1460d9b6c0) 00000100:00000001:3.0:1787695771.590728:0:52523:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787695771.590729:0:52523:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930152704 : -128555779398912 : ffff8b14466f6300) 00000100:00000001:3.0:1787695771.590734:0:52523:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695771.590736:0:52523:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695771.590738:0:52523:0:(client.c:2688:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1787695771.590740:0:52523:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1787695771.590748:0:52523:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6300 x1874531139520384/t0(0) o41->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695771.590755:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695771.590757:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695771.590760:0:52523:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff8b14466f6300 x1874531139520384/t0(0) o41->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695771.590765:0:52523:0:(client.c:1850:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695771.590766:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695771.590767:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695771.590769:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695771.590771:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695771.590773:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695771.590774:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695771.590775:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695771.590777:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695772.022506:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695772.022517:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695772.022519:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695772.022523:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695772.022526:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695772.022530:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695772.022532:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695772.214441:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695772.214446:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695772.214450:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695772.214453:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695772.214455:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695772.214457:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695772.214459:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695772.214461:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695772.214462:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695772.214464:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695772.214466:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695772.214467:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695772.214470:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695772.214471:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695772.214473:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695772.214473:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695772.214476:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695772.214477:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695772.214524:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695772.214527:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695772.214530:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1787695772.534479:0:41120:0:(connection.c:75:cpu_latency_work()) work item of ffff8b1460486220 (cpu 2) has reached its deadline 4298332473, at 4298332672 00000100:00000010:2.0:1787695772.534494:0:41120:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8b1446635840. 00000100:00000040:3.0:1787695772.534498:0:41781:0:(connection.c:75:cpu_latency_work()) work item of ffff8b1460486330 (cpu 3) has reached its deadline 4298332473, at 4298332672 00000100:00000010:3.0:1787695772.534511:0:41781:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8b1446635240. 00000100:00000001:3.0:1787695772.598445:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695772.598454:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695772.598479:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695772.598486:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695772.598491:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695772.598496:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695772.598500:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695772.598504:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695772.598508:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695773.046500:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695773.046516:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695773.046518:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695773.046524:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695773.046527:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695773.046532:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695773.046534:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695773.238483:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695773.238492:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695773.238494:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695773.238498:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695773.238503:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695773.238506:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695773.238508:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695773.238511:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695773.238515:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695773.238516:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695773.238521:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695773.238522:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695773.238526:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695773.238528:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695773.238531:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695773.238533:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695773.238536:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695773.238537:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695773.238639:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695773.238643:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695773.238645:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695773.622459:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695773.622465:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695773.622467:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695773.622472:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695773.622475:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695773.622479:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695773.622481:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695773.622483:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695773.622485:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695774.070444:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695774.070454:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695774.070456:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695774.070476:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695774.070478:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695774.070482:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695774.070484:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695774.262443:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695774.262449:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695774.262452:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695774.262453:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695774.262456:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695774.262457:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695774.262460:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695774.262460:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695774.262462:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695774.262463:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695774.262466:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695774.262467:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695774.262522:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695774.262525:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695774.262528:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695774.646432:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695774.646439:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695774.646441:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695774.646444:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695774.646447:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695774.646450:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695774.646452:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695774.646454:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695774.646456:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695775.094486:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695775.094501:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695775.094504:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695775.094509:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695775.094511:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695775.094516:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695775.094519:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695775.286487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695775.286493:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695775.286498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695775.286499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695775.286501:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695775.286502:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695775.286505:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695775.286506:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695775.286508:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695775.286509:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695775.286511:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695775.286512:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695775.286560:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695775.286564:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695775.286566:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695775.670474:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695775.670483:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695775.670485:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695775.670491:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695775.670494:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695775.670498:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695775.670501:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695775.670504:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695775.670507:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695776.118434:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695776.118446:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695776.118448:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695776.118453:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695776.118456:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695776.118460:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695776.118462:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695776.118485:0:39415:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695776.118490:0:39415:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1787695776.118493:0:39415:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695776.118495:0:39415:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1787695776.118497:0:39415:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695776.118500:0:39415:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1787695776.118506:0:39415:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665500. 00000020:00000040:1.0:1787695776.118510:0:39415:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=23 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000100:00000001:1.0:1787695776.118514:0:39415:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695776.118527:0:39415:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695776.118531:0:39415:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695776.118538:0:39415:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b1445678f00. 00000100:00000001:1.0:1787695776.118545:0:39415:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695776.118561:0:39415:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID req@ffff8b1446665500 x1874531139520512/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695776.118570:0:39415:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695776.118574:0:39415:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695776.118579:0:39415:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004ae1679a] to pc [ptlrpcd_00_00+0] req@ffff8b1446665500 x1874531139520512/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695776.118599:0:39415:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695776.118601:0:39415:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695776.118604:0:39415:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1787695776.118606:0:39415:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695776.118608:0:39415:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1787695776.118611:0:39415:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695776.118613:0:39415:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1787695776.118616:0:39415:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695776.118618:0:39415:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1787695776.118621:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:1.0:1787695776.118621:0:39415:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446667100. 00000100:00000001:0.0:1787695776.118626:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000040:1.0:1787695776.118627:0:39415:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=24 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000001:0.0:1787695776.118630:0:2403:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787695776.118631:0:39415:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695776.118633:0:39415:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695776.118635:0:39415:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695776.118639:0:2403:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665500 x1874531139520512/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1787695776.118640:0:39415:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b1445678900. 00000100:00000001:1.0:1787695776.118646:0:39415:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695776.118652:0:2403:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1787695776.118654:0:39415:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID req@ffff8b1446667100 x1874531139520640/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695776.118655:0:2403:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695776.118662:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695776.118664:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695776.118668:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695776.118676:0:2403:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2403:1874531139520512:192.168.201.140@tcp:400:kworker.0 00000100:00000001:1.0:1787695776.118678:0:39415:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695776.118682:0:39415:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695776.118683:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1787695776.118689:0:39415:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f24be523] to pc [ptlrpcd_00_01+1] req@ffff8b1446667100 x1874531139520640/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787695776.118690:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695776.118693:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695776.118697:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787695776.118702:0:39415:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695776.118704:0:39415:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1787695776.118706:0:39415:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:1.0:1787695776.118709:0:39415:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1787695776.118712:0:39415:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 02000000:00000010:0.0:1787695776.118713:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b1447625a00. 00000100:00000040:1.0:1787695776.118716:0:39415:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3673) 02000000:00000001:0.0:1787695776.118725:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695776.118729:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695776.118733:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1787695776.118735:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695776.118741:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b630. 00000100:00000001:1.0:1787695776.118746:0:2404:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000400:00000010:0.0:1787695776.118749:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96aa0. 00000100:00000040:1.0:1787695776.118753:0:2404:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446667100 x1874531139520640/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695776.118759:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874531139520512, portal 10 00000100:00000001:0.0:1787695776.118764:0:2403:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695776.118766:0:2404:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695776.118767:0:2403:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929559296 : -128555779992320 : ffff8b1446665500) 00000100:00000001:1.0:1787695776.118770:0:2404:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695776.118775:0:2404:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1787695776.118778:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665500 x1874531139520512/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695776.118780:0:2404:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695776.118785:0:2404:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695776.118789:0:2404:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446667100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:608c526c-eb6e-4a54-80a5-8969828241eb:2404:1874531139520640:192.168.201.140@tcp:400:kworker.0 00000100:00000001:1.0:1787695776.118791:0:2404:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695776.118793:0:2404:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695776.118796:0:2404:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695776.118799:0:2404:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695776.118803:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b1447533200. 00000100:00000001:0.0:1787695776.118804:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1787695776.118807:0:2404:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695776.118811:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695776.118814:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695776.118815:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695776.118820:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874531139520512, offset 0 00000400:00000010:1.0:1787695776.118824:0:2404:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702b00. 00000400:00000200:0.0:1787695776.118829:0:2403:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695776.118831:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46770. 00000100:00000200:1.0:1787695776.118836:0:2404:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874531139520640, portal 10 00000400:00000200:0.0:1787695776.118839:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695776.118839:0:2404:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695776.118841:0:2404:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929566464 : -128555779985152 : ffff8b1446667100) 00000400:00000200:0.0:1787695776.118847:0:2403:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000040:1.0:1787695776.118850:0:2404:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446667100 x1874531139520640/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695776.118854:0:2403:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1787695776.118860:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:0.0:1787695776.118861:0:2403:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000040:1.0:1787695776.118865:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695776.118869:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46330. 00000100:00000200:1.0:1787695776.118871:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874531139520640, offset 0 00000400:00000200:0.0:1787695776.118873:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695776.118877:0:2404:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695776.118881:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695776.118885:0:2403:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16876:16876:256:4294967295] 192.168.201.140@tcp LPNI seq info [16875:16875:8:4294967295] 00000400:00000200:0.0:1787695776.118889:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695776.118894:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695776.118904:0:2403:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000400:00000200:1.0:1787695776.118904:0:2404:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695776.118912:0:2404:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000800:00000200:0.0:1787695776.118917:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695776.118921:0:2404:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000800:00000010:0.0:1787695776.118922:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da700. 00000400:00000200:1.0:1787695776.118927:0:2404:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000800:00000200:0.0:1787695776.118931:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (5) 00000400:00000200:1.0:1787695776.118934:0:2404:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695776.118942:0:2404:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16877:16877:255:4294967295] 192.168.201.140@tcp LPNI seq info [16876:16876:7:4294967295] 00000800:00000200:0.0:1787695776.118943:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.140@tcp ip 192.168.201.140:988 00000800:00000200:0.0:1787695776.118949:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8b14460da700 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1787695776.118955:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695776.118965:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695776.118966:0:2404:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695776.118973:0:2404:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000040:0.0:1787695776.118974:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000800:00000010:1.0:1787695776.118978:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1445678300. 00000400:00000200:0.0:1787695776.118982:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000800:00000200:1.0:1787695776.118987:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (5) 00000100:00000001:0.0:1787695776.118992:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298336581 00000100:00000010:0.0:1787695776.118998:0:2403:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8b14466355a0. 00000800:00000200:1.0:1787695776.118998:0:2404:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.140@tcp ip 192.168.201.140:988 00000800:00000200:1.0:1787695776.119005:0:2404:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8b1445678300 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1787695776.119006:0:2403:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request now active for cpu 2 00000100:00000001:0.0:1787695776.119013:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298336581 00000100:00000001:1.0:1787695776.119013:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1787695776.119013:0:2398:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1787695776.119020:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da700. 00000100:00000010:0.0:1787695776.119021:0:2403:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8b1446635300. 00000100:00000040:1.0:1787695776.119022:0:2404:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000100:00000001:0.0:1787695776.119028:0:2403:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request now active for cpu 3 00000400:00000200:2.0:1787695776.119029:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1787695776.119033:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695776.119034:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695776.119038:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298336447 00000100:00000001:0.0:1787695776.119039:0:2403:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695776.119040:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: OK 00000100:00000001:1.0:1787695776.119045:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298336581 00000100:00000001:0.0:1787695776.119047:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695776.119048:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000100:00000001:0.0:1787695776.119049:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1787695776.119051:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695776.119055:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695776.119057:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=49 : 49 : 31) 00000100:00000001:0.0:1787695776.119062:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695776.119063:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298336447 00000100:00000001:0.0:1787695776.119065:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695776.119066:0:2398:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787695776.119067:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298336581 00000100:00000001:0.0:1787695776.119068:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695776.119073:0:2404:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:2.0:1787695776.119074:0:2398:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b1446665500 x1874531139520512/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695776.119075:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695776.119080:0:2404:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695776.119089:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695776.119092:0:2398:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787695776.119094:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695776.119097:0:2398:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665500 x1874531139520512/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695861 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695776.119102:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695776.119104:0:2398:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695776.119106:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=39 : 39 : 27) 00000100:00000001:2.0:1787695776.119107:0:2398:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695776.119112:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695776.119115:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695776.119118:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1787695776.119120:0:2398:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1787695776.119121:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1787695776.119123:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1445678300. 00000400:00000200:2.0:1787695776.119136:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1787695776.119140:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: OK 00000400:00000200:2.0:1787695776.119142:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46330 00000400:00000010:2.0:1787695776.119143:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46330. 00000100:00000001:2.0:1787695776.119146:0:2398:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695776.119149:0:2398:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b1446667100 x1874531139520640/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695776.119154:0:2398:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695776.119156:0:2398:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446667100 x1874531139520640/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695861 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695776.119161:0:2398:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695776.119162:0:2398:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1787695776.310463:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695776.310471:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695776.310477:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695776.310479:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695776.310483:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695776.310486:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695776.310489:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695776.310495:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695776.310497:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695776.310499:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695776.310502:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695776.310505:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695776.310507:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695776.310510:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695776.310514:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695776.310516:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695776.310517:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695776.310519:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695776.310520:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695776.310521:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695776.310523:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695776.310534:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695776.310536:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695776.310538:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695776.310543:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695776.310561:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695776.310565:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695776.310567:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695776.310569:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695776.310572:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695776.310574:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695776.310575:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695776.310577:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695776.310578:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695776.310580:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695776.310582:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695776.310625:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695776.310628:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695776.310631:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1787695776.446425:0:41120:0:(connection.c:75:cpu_latency_work()) work item of ffff8b1460486220 (cpu 2) has reached its deadline 4298336581, at 4298336584 00000100:00000010:2.0:1787695776.446438:0:41120:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8b14466355a0. 00000100:00000040:3.0:1787695776.446444:0:41781:0:(connection.c:75:cpu_latency_work()) work item of ffff8b1460486330 (cpu 3) has reached its deadline 4298336581, at 4298336584 00000100:00000010:3.0:1787695776.446467:0:41781:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8b1446635300. 00000100:00000001:3.0:1787695776.694518:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695776.694527:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695776.694529:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695776.694534:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695776.694538:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695776.694541:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695776.694544:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695776.694546:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695776.694549:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695777.142497:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695777.142512:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695777.142514:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695777.142520:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695777.142522:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695777.142528:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695777.142530:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695777.334459:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695777.334468:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695777.334475:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695777.334477:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695777.334481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695777.334483:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695777.334486:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695777.334488:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695777.334491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695777.334492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695777.334495:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695777.334496:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695777.334559:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695777.334562:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695777.334565:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695777.718458:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695777.718468:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695777.718472:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695777.718479:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695777.718484:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695777.718489:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695777.718493:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695777.718495:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695777.718498:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695778.166463:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695778.166479:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695778.166482:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695778.166488:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695778.166490:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695778.166496:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695778.166499:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695778.358440:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695778.358446:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695778.358450:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695778.358451:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695778.358454:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695778.358455:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695778.358457:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695778.358458:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695778.358461:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695778.358461:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695778.358464:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695778.358464:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695778.358518:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695778.358520:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695778.358523:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695778.742481:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695778.742490:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695778.742492:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695778.742498:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695778.742502:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695778.742506:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695778.742509:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695778.742512:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695778.742514:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695779.190544:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695779.190564:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695779.190567:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695779.190575:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695779.190578:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695779.190585:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695779.190588:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695779.382482:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695779.382488:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695779.382493:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695779.382510:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695779.382513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695779.382514:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695779.382517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695779.382518:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695779.382520:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695779.382521:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695779.382523:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695779.382524:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695779.382573:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695779.382576:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695779.382578:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695779.766519:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695779.766530:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695779.766533:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695779.766541:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695779.766546:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695779.766551:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695779.766588:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695779.766597:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695779.766602:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695780.214429:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695780.214435:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695780.214451:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695760/real 1787695760] req@ffff8b1446607b80 x1874531139518976/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695780 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695780.214472:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695780.214475:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695780.214479:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bf20. 00000400:00000200:0.0:1787695780.214488:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96d48 00000400:00000010:0.0:1787695780.214492:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96d48. 00000100:00000001:0.0:1787695780.214497:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695780.214502:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446607b80 x1874531139518976/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695780 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695780.214509:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446607b80 x1874531139518976/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695780 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695780.214515:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695780.214517:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695780.214519:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695780.214520:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695780.214523:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446607b80 x1874531139518976/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695780 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695780.214528:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695780.214535:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695780.214536:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695780.214554:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695780.214556:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695780.214559:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695780.214563:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446607b80 x1874531139518976/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695780 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695780.214568:0:2402:0:(client.c:2142:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695780.214570:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695780.214571:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695780.214572:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695780.214574:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695780.214576:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695780.214578:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695780.214601:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695780.214603:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695780.214606:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446607b80 x1874531139518976/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695780 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695780.214612:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446607b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139518976:192.168.201.140@tcp:250:kworker.0 00000100:00000001:0.0:1787695780.214619:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695780.214622:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695780.214625:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446607b80 x1874531139518976/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695780 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695780.214632:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695780.214634:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695780.214638:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545800. 00000100:00080000:1.0:1787695780.214638:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1787695780.214646:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:1.0:1787695780.214651:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000020:00000001:0.0:1787695780.214652:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695780.214658:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000100:00000001:1.0:1787695780.214658:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695780.214666:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000020:00000001:0.0:1787695780.214667:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695780.214673:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f5800. 00000100:00000001:1.0:1787695780.214673:0:47074:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:0.0:1787695780.214683:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:1.0:1787695780.214687:0:47074:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3653 02000000:00000001:0.0:1787695780.214688:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695780.214696:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446607b80. 00000100:00000001:0.0:1787695780.214700:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000400:00000200:1.0:1787695780.214701:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695780.214702:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695780.214706:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695780.214710:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00001000:1.0:1787695780.214715:0:47074:0:(import.c:2149:obd_at_measure()) add 20 to ffff8b1442fe5418 time=103 v=15 (15 0 0 0) 00000100:00001000:1.0:1787695780.214718:0:47074:0:(import.c:2215:obd_at_measure()) AT ffff8b1442fe5418 change: old=15 new=20 delta=5 (val=20) hist 20 0 0 0 00000100:00080000:1.0:1787695780.214721:0:47074:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 15s 00000100:00000040:1.0:1787695780.214726:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695780.214729:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695780.214733:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695780.214735:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695780.214738:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695780.214741:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:1.0:1787695780.214744:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695780.214747:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:1.0:1787695780.214749:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:1.0:1787695780.214752:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:1.0:1787695780.214756:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695780.214759:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695780.214779:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695780.214782:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695780.214786:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:1.0:1787695780.214791:0:47074:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695780.214794:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695780.214798:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695780.214802:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695780.214806:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695780.214808:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695780.214811:0:47074:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695780.214818:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446664380. 00000020:00000040:1.0:1787695780.214821:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:1.0:1787695780.214825:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695780.214826:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695780.214830:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695780.214832:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695780.214836:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695780.214843:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446544400. 00000100:00000001:1.0:1787695780.214850:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695780.214858:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446664380 x1874531139520768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695780.214867:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695780.214871:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695780.214874:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000094a36b85] to pc [ptlrpcd_rcv+-1] req@ffff8b1446664380 x1874531139520768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695780.214893:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695780.214895:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695780.214898:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1787695780.214901:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695780.214904:0:47074:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1787695780.214912:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446667b80. 00000020:00000040:1.0:1787695780.214915:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=24 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000100:00000001:1.0:1787695780.214919:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695780.214921:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695780.214923:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695780.214929:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b1445678000. 00000100:00000001:0.0:1787695780.214930:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695780.214933:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695780.214934:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695780.214935:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695780.214942:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446664380 x1874531139520768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695780.214948:0:47074:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID req@ffff8b1446667b80 x1874531139520896/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695780.214958:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695780.214960:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695780.214964:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695780.214966:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695780.214979:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695780.214989:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1787695780.214992:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446664380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139520768:192.168.201.140@tcp:250:kworker.0 00000100:00000001:1.0:1787695780.214992:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695780.215000:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006b2dcf49] to pc [ptlrpcd_00_00+0] req@ffff8b1446667b80 x1874531139520896/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695780.215003:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695780.215013:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695780.215018:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695780.215021:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695780.215027:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14475fa400. 02000000:00000001:0.0:1787695780.215030:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695780.215036:0:47074:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695780.215039:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:1.0:1787695780.215045:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:0.0:1787695780.215046:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bf20. 00000400:00000010:0.0:1787695780.215054:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96d48. 00000100:00080000:1.0:1787695780.215056:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000200:0.0:1787695780.215061:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139520768, portal 25 00000100:00080000:1.0:1787695780.215061:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695780.215069:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695780.215074:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929554816 : -128555779996800 : ffff8b1446664380) 00000100:00080000:1.0:1787695780.215075:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1787695780.215081:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1787695780.215082:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446664380 x1874531139520768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695805 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695780.215100:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000001:0.0:1787695780.215102:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695780.215108:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1787695780.215109:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695780.215113:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96990. 00000100:00000001:1.0:1787695780.215118:0:47074:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000200:0.0:1787695780.215120:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139520768, offset 0 00000100:00000010:1.0:1787695780.215123:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665f80. 00000020:00000040:1.0:1787695780.215127:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=25 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000400:00000200:0.0:1787695780.215128:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787695780.215131:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695780.215133:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695780.215137:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695780.215152:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000010:1.0:1787695780.215155:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b1445678800. 00000100:00000001:1.0:1787695780.215159:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695780.215162:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695780.215168:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:1.0:1787695780.215168:0:47074:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID req@ffff8b1446665f80 x1874531139521024/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1787695780.215179:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695780.215192:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1787695780.215200:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1787695780.215204:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:1.0:1787695780.215208:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695780.215231:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a0a2be0e] to pc [ptlrpcd_00_01+1] req@ffff8b1446665f80 x1874531139521024/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695780.215232:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16878:16878:256:4294967295] 192.168.201.140@tcp LPNI seq info [16877:16877:8:4294967295] 00000400:00000200:0.0:1787695780.215255:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695780.215265:0:47074:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695780.215269:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1787695780.215283:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:0.0:1787695780.215284:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000040:1.0:1787695780.215295:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:0.0:1787695780.215304:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000040:1.0:1787695780.215304:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:1.0:1787695780.215316:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3678) 00000800:00000010:0.0:1787695780.215317:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460daa00. 00000800:00000200:0.0:1787695780.215331:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (5) 00000100:00000001:1.0:1787695780.215333:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695780.215337:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695780.215338:0:2403:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000800:00000200:0.0:1787695780.215346:0:2402:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.140@tcp ip 192.168.201.140:988 00000100:00000040:1.0:1787695780.215347:0:2403:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446667b80 x1874531139520896/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1787695780.215361:0:2402:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8b14460daa00 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:1.0:1787695780.215392:0:2403:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695780.215393:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695780.215396:0:2403:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695780.215401:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:1.0:1787695780.215402:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787695780.215405:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298351929 02000000:00000001:1.0:1787695780.215406:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695780.215420:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1787695780.215422:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8b1446635cc0. 00000100:00100000:1.0:1787695780.215429:0:2403:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446667b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2403:1874531139520896:192.168.201.140@tcp:400:kworker.0 00000100:00000001:1.0:1787695780.215433:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787695780.215434:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request now active for cpu 2 02000000:00000001:1.0:1787695780.215439:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787695780.215441:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298351929 02000000:00000001:1.0:1787695780.215442:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695780.215449:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:0.0:1787695780.215452:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8b1446635120. 00000800:00000200:2.0:1787695780.215460:0:2397:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1787695780.215462:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request now active for cpu 3 02000000:00000010:1.0:1787695780.215462:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b1447532c00. 02000000:00000001:1.0:1787695780.215471:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1787695780.215473:0:2397:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460daa00. 00000100:00000001:0.0:1787695780.215477:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1787695780.215481:0:2397:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1787695780.215486:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695780.215492:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695780.215493:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702630. 00000400:00000200:2.0:1787695780.215501:0:2397:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: OK 00000100:00000001:0.0:1787695780.215506:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695780.215507:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000400:00000200:2.0:1787695780.215510:0:2397:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96990 00000100:00000001:0.0:1787695780.215516:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1787695780.215517:0:2397:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96990. 00000100:00000200:1.0:1787695780.215518:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874531139520896, portal 10 00000100:00000001:0.0:1787695780.215522:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695780.215524:0:2403:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695780.215526:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695780.215527:0:2403:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929569152 : -128555779982464 : ffff8b1446667b80) 00000100:00000001:2.0:1787695780.215528:0:2397:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787695780.215531:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695780.215535:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695780.215537:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695780.215539:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1787695780.215542:0:2397:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b1446664380 x1874531139520768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695805 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695780.215552:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446667b80 x1874531139520896/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695865 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695780.215568:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695780.215575:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695780.215583:0:2404:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695780.215596:0:2404:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665f80 x1874531139521024/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695780.215610:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787695780.215622:0:2397:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695780.215625:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695780.215633:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46f68. 00000100:00000001:0.0:1787695780.215634:0:2404:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:2.0:1787695780.215634:0:2397:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446664380 x1874531139520768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695805 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695780.215638:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874531139520896, offset 0 00000100:00000001:0.0:1787695780.215639:0:2404:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695780.215658:0:2404:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787695780.215659:0:2397:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695780.215665:0:2403:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 02000000:00000001:0.0:1787695780.215667:0:2404:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695780.215669:0:2397:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1787695780.215677:0:2404:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695780.215684:0:2404:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:608c526c-eb6e-4a54-80a5-8969828241eb:2404:1874531139521024:192.168.201.140@tcp:400:kworker.0 00000400:00000200:1.0:1787695780.215685:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695780.215694:0:2404:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:1.0:1787695780.215697:0:2403:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 02000000:00000001:0.0:1787695780.215698:0:2404:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695780.215707:0:2404:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695780.215710:0:2403:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1787695780.215712:0:2404:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695780.215717:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b1447625600. 00000400:00000200:1.0:1787695780.215718:0:2403:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 02000000:00000001:0.0:1787695780.215725:0:2404:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695780.215729:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695780.215739:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695780.215740:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695780.215748:0:2404:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b2c0. 00000400:00000200:1.0:1787695780.215749:0:2403:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16879:16879:256:4294967295] 192.168.201.140@tcp LPNI seq info [16878:16878:8:4294967295] 00000400:00000010:0.0:1787695780.215756:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f964c8. 00000400:00000200:1.0:1787695780.215761:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000200:0.0:1787695780.215762:0:2404:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874531139521024, portal 10 00000100:00000001:0.0:1787695780.215768:0:2404:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695780.215780:0:2404:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929561984 : -128555779989632 : ffff8b1446665f80) 00000400:00000200:1.0:1787695780.215789:0:2403:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000040:0.0:1787695780.215795:0:2404:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665f80 x1874531139521024/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695865 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695780.215807:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000200:1.0:1787695780.215812:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000040:0.0:1787695780.215820:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695780.215823:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96c38. 00000800:00000010:1.0:1787695780.215823:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1445678300. 00000100:00000200:0.0:1787695780.215826:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874531139521024, offset 0 00000800:00000200:1.0:1787695780.215836:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (5) 00000400:00000200:0.0:1787695780.215837:0:2404:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000800:00000200:1.0:1787695780.215854:0:2403:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.140@tcp ip 192.168.201.140:988 00000400:00000200:0.0:1787695780.215855:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000800:00000200:1.0:1787695780.215866:0:2403:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8b1445678300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1787695780.215870:0:2404:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695780.215875:0:2404:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695780.215885:0:2404:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:1.0:1787695780.215887:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695780.215891:0:2404:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000040:1.0:1787695780.215895:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:0.0:1787695780.215898:0:2404:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695780.215907:0:2404:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16880:16880:255:4294967295] 192.168.201.140@tcp LPNI seq info [16879:16879:7:4294967295] 00000400:00000200:1.0:1787695780.215908:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695780.215916:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298340678 00000400:00000200:0.0:1787695780.215923:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695780.215924:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298351929 00000100:00000001:1.0:1787695780.215930:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298340678 00000100:00000001:1.0:1787695780.215934:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298351929 00000100:00000001:1.0:1787695780.215944:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.0:1787695780.215948:0:2398:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000400:00000200:0.0:1787695780.215949:0:2404:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:1.0:1787695780.215957:0:2403:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1787695780.215959:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1445678300. 00000800:00000200:0.0:1787695780.215969:0:2404:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787695780.215972:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695780.215973:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000800:00000010:0.0:1787695780.215974:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da500. 00000100:00000001:1.0:1787695780.215979:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695780.215989:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:2.0:1787695780.215989:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: OK 00000100:00000001:1.0:1787695780.215997:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=45 : 45 : 2d) 00000800:00000200:0.0:1787695780.216000:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (5) 00000400:00000200:2.0:1787695780.216000:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46f68 00000100:00000001:1.0:1787695780.216001:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695780.216006:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000010:2.0:1787695780.216010:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46f68. 00000100:00000001:1.0:1787695780.216012:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695780.216019:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695780.216019:0:2398:0:(events.c:34:request_out_callback()) Process entered 00000800:00000200:0.0:1787695780.216029:0:2404:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.140@tcp ip 192.168.201.140:988 00000100:00000200:2.0:1787695780.216030:0:2398:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b1446667b80 x1874531139520896/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695865 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1787695780.216038:0:2404:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8b14460da500 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1787695780.216053:0:2398:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787695780.216054:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695780.216062:0:2404:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000100:00000040:2.0:1787695780.216062:0:2398:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446667b80 x1874531139520896/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695865 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695780.216081:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695780.216085:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298340544 00000100:00000001:0.0:1787695780.216088:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298351929 00000100:00000001:0.0:1787695780.216090:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298340544 00000100:00000001:0.0:1787695780.216092:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298351929 00000100:00000001:0.0:1787695780.216097:0:2404:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695780.216102:0:2404:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695780.216105:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695780.216108:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695780.216108:0:2398:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695780.216120:0:2398:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695780.216123:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695780.216125:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=35 : 35 : 23) 00000100:00000001:0.0:1787695780.216128:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695780.216129:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695780.216133:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695780.216135:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1787695780.216145:0:2398:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(0) 0 00000800:00000010:2.0:1787695780.216150:0:2398:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da500. 00000400:00000200:2.0:1787695780.216155:0:2398:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1787695780.216161:0:2398:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: OK 00000400:00000200:2.0:1787695780.216164:0:2398:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96c38 00000400:00000010:2.0:1787695780.216167:0:2398:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96c38. 00000100:00000001:2.0:1787695780.216170:0:2398:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695780.216175:0:2398:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff8b1446665f80 x1874531139521024/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695865 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695780.216184:0:2398:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695780.216188:0:2398:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665f80 x1874531139521024/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695865 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695780.216205:0:2398:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695780.216207:0:2398:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1787695780.406490:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695780.406501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695780.406509:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695780.406510:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695780.406513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695780.406514:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695780.406517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695780.406518:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695780.406521:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695780.406522:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695780.406525:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695780.406525:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695780.406579:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695780.406586:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695780.406589:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695780.790447:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695780.790454:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695780.790456:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695780.790460:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695780.790463:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695780.790466:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695780.790468:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695780.790470:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695780.790472:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695781.238473:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695781.238485:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695781.238486:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695781.238505:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695781.238507:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695781.238510:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695781.238512:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695781.430474:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695781.430479:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695781.430483:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695781.430484:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695781.430487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695781.430488:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695781.430490:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695781.430491:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695781.430508:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695781.430511:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695781.430513:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695781.430517:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695781.430519:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695781.430520:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695781.430522:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695781.430524:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695781.430526:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695781.430527:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695781.430529:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695781.430530:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695781.430580:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695781.430584:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695781.430586:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695781.814495:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695781.814505:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695781.814508:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695781.814515:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695781.814520:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695781.814524:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695781.814527:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695781.814531:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695781.814534:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695782.262473:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695782.262485:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695782.262486:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695782.262491:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695782.262492:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695782.262496:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695782.262498:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695782.454456:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695782.454463:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695782.454470:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695782.454473:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695782.454475:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695782.454479:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695782.454482:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695782.454484:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695782.454486:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695782.454488:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695782.454492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695782.454494:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695782.454497:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695782.454499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695782.454502:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695782.454504:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695782.454508:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695782.454509:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695782.454553:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695782.454556:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695782.454559:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695782.838446:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695782.838453:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695782.838454:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695782.838458:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695782.838461:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695782.838464:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695782.838466:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695782.838468:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695782.838470:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695783.286460:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695783.286472:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695783.286473:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695783.286479:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695783.286481:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695783.286484:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695783.286486:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695783.478479:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695783.478487:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695783.478489:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695783.478491:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695783.478496:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695783.478497:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695783.478499:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695783.478501:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695783.478518:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695783.478520:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695783.478522:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695783.478523:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695783.478525:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695783.478526:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695783.478529:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695783.478530:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695783.478532:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695783.478533:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695783.478589:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695783.478593:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695783.478596:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000200:3.0:1787695783.606454:0:2394:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (5) 00000800:00000200:3.0:1787695783.606463:0:2394:0:(socklnd_cb.c:678:ksocknal_queue_tx_locked()) Sending to 12345-192.168.201.140@tcp ip 192.168.201.140:988 00000800:00000200:3.0:1787695783.606481:0:2394:0:(socklnd_cb.c:697:ksocknal_queue_tx_locked()) Packet ffff8b1458095500 type 192, nob 24 niov 1 nkiov 0 00000800:00000100:3.0:1787695783.606497:0:2394:0:(socklnd_cb.c:2254:ksocknal_find_timed_out_conn()) A connection with 12345-192.168.201.140@tcp (192.168.201.140:988) timed out; the network or node may be down. 00000800:00000200:3.1F:1787695783.606501:0:2394:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 3, ksnr_connected 6 ksnr_max_conns 1 00000800:00000200:3.1:1787695783.606508:0:2394:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 2, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:3.1:1787695783.606510:0:2394:0:(socklnd.c:424:ksocknal_decr_conn_count()) Del conn type 1, ksnr_connected 0 ksnr_max_conns 1 00000400:00000200:3.0:1787695783.606520:0:2394:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000800:00000200:2.0:1787695783.606521:0:2397:0:(socklnd_cb.c:469:ksocknal_process_transmit()) send(24) -108 00000400:00000200:3.0:1787695783.606525:0:2394:0:(router.c:1791:lnet_notify()) 192.168.201.40@tcp notifying 192.168.201.140@tcp: down 00000400:00000200:3.0:1787695783.606531:0:2394:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000800:00000200:3.0:1787695783.606536:0:2394:0:(socklnd.c:1681:ksocknal_notify_gw_down()) gw 192.168.201.140@tcp down 00000800:00000200:3.0:1787695783.606619:0:2394:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection ffff8b1442eb6000 00000800:00000010:3.0:1787695783.606623:0:2394:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at ffff8b1442eb6000. 00000800:00000200:3.0:1787695783.606628:0:2394:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection ffff8b1443344000 00000800:00000010:3.0:1787695783.606630:0:2394:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at ffff8b1443344000. 00000800:00000200:3.0:1787695783.606634:0:2394:0:(socklnd.c:1528:ksocknal_destroy_conn()) connection ffff8b1443346000 00000800:00000010:3.0:1787695783.606635:0:2394:0:(socklnd.c:1581:ksocknal_destroy_conn()) kfreed 'conn': 4728 at ffff8b1443346000. 00000100:00000001:3.0:1787695783.862479:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695783.862485:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695783.862487:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695783.862492:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695783.862495:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695783.862498:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695783.862500:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695783.862502:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695783.862504:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695784.310430:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695784.310440:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695784.310456:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695759/real 1787695759] req@ffff8b1446605880 x1874531139518720/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695784 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695784.310473:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695784.310476:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695784.310479:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b7e8. 00000400:00000200:0.0:1787695784.310484:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96660 00000400:00000010:0.0:1787695784.310487:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96660. 00000100:00000001:0.0:1787695784.310490:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695784.310494:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446605880 x1874531139518720/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695784 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695784.310501:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446605880 x1874531139518720/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695784 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695784.310532:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695784.310534:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695784.310537:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695784.310539:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695784.310543:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446605880 x1874531139518720/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695784 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695784.310548:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695784.310550:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695784.310554:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695759/real 1787695759] req@ffff8b1446606300 x1874531139518464/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695784 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695784.310560:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695784.310560:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695784.310562:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b8f0. 00000400:00000200:0.0:1787695784.310563:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96198 00000400:00000010:0.0:1787695784.310564:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96198. 00000100:00000001:0.0:1787695784.310566:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695784.310568:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446606300 x1874531139518464/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695784 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695784.310574:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446606300 x1874531139518464/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695784 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695784.310578:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695784.310579:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695784.310585:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695784.310586:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695784.310588:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446606300 x1874531139518464/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695784 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695784.310593:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695784.310601:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695784.310603:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695784.310609:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695784.310611:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695784.310615:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695784.310619:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446605880 x1874531139518720/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695784 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695784.310629:0:2402:0:(client.c:2142:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695784.310632:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695784.310634:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695784.310636:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695784.310639:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695784.310642:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695784.310646:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695784.310673:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695784.310677:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695784.310682:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446605880 x1874531139518720/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695784 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695784.310692:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446605880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139518720:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695784.310698:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695784.310701:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446606300 x1874531139518464/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695784 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695784.310706:0:2402:0:(client.c:2142:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695784.310709:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695784.310710:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695784.310711:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695784.310718:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695784.310720:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695784.310722:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695784.310728:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695784.310736:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695784.310740:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446606300 x1874531139518464/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695784 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695784.310745:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787695784.310747:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446606300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139518464:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695784.310760:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695784.310762:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000001:0.0:1787695784.310765:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1787695784.310766:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1787695784.310775:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446605880 x1874531139518720/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695784 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695784.310775:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787695784.310785:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695784.310791:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695784.310796:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1787695784.310799:0:47074:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:0.0:1787695784.310800:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695784.310805:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547c00. 00000100:00080000:1.0:1787695784.310808:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3652 02000000:00000001:0.0:1787695784.310814:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695784.310820:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000400:00000200:1.0:1787695784.310821:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000040:0.0:1787695784.310829:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695784.310833:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000400:00000200:1.0:1787695784.310834:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 02000000:00000010:0.0:1787695784.310837:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f5c00. 00000100:00080000:1.0:1787695784.310838:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 02000000:00000001:0.0:1787695784.310846:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:1.0:1787695784.310847:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695784.310855:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787695784.310856:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000010:0.0:1787695784.310862:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446605880. 00000100:00000040:1.0:1787695784.310866:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695784.310870:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695784.310870:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695784.310875:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787695784.310877:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787695784.310879:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000040:1.0:1787695784.310882:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000100:00000040:0.0:1787695784.310888:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446606300 x1874531139518464/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695784 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787695784.310888:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695784.310894:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:1.0:1787695784.310904:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000001:0.0:1787695784.310917:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000020:00000001:1.0:1787695784.310918:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:1.0:1787695784.310942:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695784.310944:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695784.310946:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695784.310947:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695784.310949:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:1.0:1787695784.310952:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695784.310954:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695784.310955:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787695784.310961:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:0.0:1787695784.310962:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:0.0:1787695784.310969:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695784.310971:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695784.310971:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:1.0:1787695784.310973:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1787695784.310974:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695784.310979:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445435c00. 02000000:00000001:1.0:1787695784.310980:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695784.310982:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787695784.310986:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787695784.310989:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000008:00000001:1.0:1787695784.310990:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695784.310993:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787695784.310994:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1787695784.310996:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446606300. 00000100:00000001:0.0:1787695784.310999:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695784.311001:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:1.0:1787695784.311002:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446667800. 00000100:00000001:0.0:1787695784.311005:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1787695784.311007:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:1.0:1787695784.311013:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695784.311016:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695784.311020:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695784.311022:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695784.311026:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695784.311033:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446544c00. 00000100:00000001:1.0:1787695784.311039:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695784.311047:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446667800 x1874531139521152/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695784.311055:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695784.311059:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695784.311063:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009df211f4] to pc [ptlrpcd_rcv+-1] req@ffff8b1446667800 x1874531139521152/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695784.311085:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695784.311088:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695784.311096:0:47074:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1787695784.311100:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446666d80. 00000020:00000040:1.0:1787695784.311102:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=25 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000100:00000001:1.0:1787695784.311107:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695784.311108:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695784.311111:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695784.311112:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695784.311115:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695784.311119:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000010:1.0:1787695784.311119:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b1445678300. 00000100:00000001:1.0:1787695784.311128:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695784.311131:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446667800 x1874531139521152/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695784.311136:0:47074:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID req@ffff8b1446666d80 x1874531139521280/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695784.311188:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695784.311190:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695784.311190:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695784.311199:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695784.311201:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695784.311206:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1787695784.311207:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa82a36] to pc [ptlrpcd_00_00+0] req@ffff8b1446666d80 x1874531139521280/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787695784.311210:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695784.311215:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446667800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139521152:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695784.311219:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695784.311225:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695784.311228:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695784.311238:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695784.311250:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14475fbc00. 02000000:00000001:0.0:1787695784.311253:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695784.311261:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695784.311264:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b8f0. 00000100:00000001:1.0:1787695784.311269:0:47074:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695784.311270:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96198. 00000100:00080000:1.0:1787695784.311280:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:0.0:1787695784.311282:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139521152, portal 10 00000100:00080000:1.0:1787695784.311286:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000001:0.0:1787695784.311287:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695784.311290:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929568256 : -128555779983360 : ffff8b1446667800) 00000100:00080000:1.0:1787695784.311296:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1787695784.311303:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446667800 x1874531139521152/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695809 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695784.311303:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1787695784.311311:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695784.311316:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695784.311322:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00080000:1.0:1787695784.311322:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000400:00000010:0.0:1787695784.311325:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96660. 00000100:00000001:1.0:1787695784.311327:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695784.311331:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:0.0:1787695784.311335:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139521152, offset 0 00000100:00000001:1.0:1787695784.311337:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695784.311347:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3652 00000400:00000200:0.0:1787695784.311348:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695784.311357:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695784.311394:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695784.311403:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695784.311408:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000200:0.0:1787695784.311410:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695784.311415:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00080000:1.0:1787695784.311424:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1787695784.311427:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695784.311433:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000040:1.0:1787695784.311438:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695784.311448:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16881:16881:256:4294967295] 192.168.201.140@tcp LPNI seq info [16880:16880:8:4294967295] 00000100:00000001:1.0:1787695784.311448:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695784.311460:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695784.311468:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695784.311470:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695784.311474:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695784.311475:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:1.0:1787695784.311478:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695784.311480:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000020:00000001:1.0:1787695784.311489:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000200:0.0:1787695784.311493:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000020:00000001:1.0:1787695784.311495:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000040:1.0:1787695784.311505:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000800:00000200:0.0:1787695784.311506:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787695784.311516:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695784.311518:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000010:0.0:1787695784.311519:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da500. 00000100:00000001:1.0:1787695784.311523:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695784.311527:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:1.0:1787695784.311533:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000800:00000200:0.0:1787695784.311535:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:1.0:1787695784.311540:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695784.311545:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:0.1F:1787695784.311554:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:1.0:1787695784.311555:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695784.311559:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695784.311560:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787695784.311565:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787695784.311570:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695784.311571:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695784.311574:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446664000. 00000020:00000040:1.0:1787695784.311576:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:1.0:1787695784.311578:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695784.311579:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695784.311581:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787695784.311582:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695784.311583:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:0.0:1787695784.311589:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 02000000:00000001:1.0:1787695784.311591:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695784.311595:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000010:1.0:1787695784.311597:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545000. 00000100:00000001:0.0:1787695784.311600:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298345926 00000100:00000001:1.0:1787695784.311604:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695784.311605:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298351929 00000100:00000001:0.0:1787695784.311612:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298345926 00000100:00000001:0.0:1787695784.311615:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298351929 00000100:00100000:1.0:1787695784.311619:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446664000 x1874531139521408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695784.311620:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695784.311627:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695784.311637:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695784.311640:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695784.311642:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695784.311647:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695784.311651:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695784.311656:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695784.311657:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e60c2a2d] to pc [ptlrpcd_rcv+-1] req@ffff8b1446664000 x1874531139521408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695784.311662:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695784.311666:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695784.311672:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695784.311677:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695784.311678:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695784.311682:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1787695784.311684:0:2392:0:(lib-socket.c:438:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.140:988 00000100:00000001:0.0:1787695784.311692:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695784.311701:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695784.311702:0:47074:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000001:0.0:1787695784.311712:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:1.0:1787695784.311713:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446666300. 00000100:00000001:0.0:1787695784.311716:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000040:1.0:1787695784.311718:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=26 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000001:0.0:1787695784.311719:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787695784.311726:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695784.311729:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:0.0:1787695784.311730:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446664000 x1874531139521408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695784.311737:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695784.311748:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000010:1.0:1787695784.311749:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b1444663c00. 00000400:00000200:2.0:1787695784.311750:0:2392:0:(lib-socket.c:438:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.140:988 00000100:00000001:0.0:1787695784.311752:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695784.311759:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695784.311764:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695784.311766:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695784.311768:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695784.311776:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446664000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139521408:192.168.201.140@tcp:38:kworker.0 00000100:00000040:1.0:1787695784.311782:0:47074:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID req@ffff8b1446666300 x1874531139521536/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695784.311784:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695784.311790:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695784.311795:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695784.311802:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787695784.311810:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000010:0.0:1787695784.311813:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14475f9800. 00000100:00000001:1.0:1787695784.311814:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695784.311823:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695784.311832:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000945f4e20] to pc [ptlrpcd_00_01+1] req@ffff8b1446666300 x1874531139521536/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695784.311834:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695784.311845:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b7e8. 00000400:00000010:0.0:1787695784.311855:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96b28. 00000100:00000001:1.0:1787695784.311857:0:47074:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695784.311861:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139521408, portal 10 00000100:00000040:1.0:1787695784.311862:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695784.311868:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695784.311873:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929553920 : -128555779997696 : ffff8b1446664000) 00000100:00000040:1.0:1787695784.311873:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:1.0:1787695784.311887:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1787695784.311890:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446664000 x1874531139521408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695809 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695784.311907:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1787695784.311912:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695784.311920:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000040:1.0:1787695784.311921:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 1 (3682) 00000400:00000010:0.0:1787695784.311922:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f962a8. 00000100:00000200:0.0:1787695784.311928:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139521408, offset 0 00000400:00000200:0.0:1787695784.311933:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695784.311937:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695784.311943:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1787695784.311948:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:1.0:1787695784.311948:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695784.311958:0:2403:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000400:00000200:0.0:1787695784.311961:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695784.311968:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000040:1.0:1787695784.311968:0:2403:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446666d80 x1874531139521280/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695784.311974:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695784.311985:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:1.0:1787695784.311998:0:2403:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000400:00000200:0.0:1787695784.311999:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16882:16882:255:4294967295] 192.168.201.140@tcp LPNI seq info [16881:16881:7:4294967295] 00000100:00000001:1.0:1787695784.312001:0:2403:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695784.312008:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695784.312011:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1787695784.312015:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:1.0:1787695784.312017:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695784.312025:0:2403:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446666d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2403:1874531139521280:192.168.201.140@tcp:400:kworker.0 00000100:00000001:1.0:1787695784.312037:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:0.0:1787695784.312039:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 02000000:00000001:1.0:1787695784.312044:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695784.312048:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1787695784.312049:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 02000000:00000001:1.0:1787695784.312060:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000800:00000010:0.0:1787695784.312061:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460daa00. 02000000:00000010:1.0:1787695784.312065:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b1447532e00. 00000800:00000200:0.0:1787695784.312075:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:1.0:1787695784.312075:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695784.312094:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000800:00000200:0.1:1787695784.312096:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000400:00000010:1.0:1787695784.312101:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702318. 00000100:00000001:0.0:1787695784.312102:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695784.312117:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000010:1.0:1787695784.312119:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46220. 00000400:00000200:0.0:1787695784.312123:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695784.312125:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298344662 00000100:00000200:1.0:1787695784.312126:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874531139521280, portal 10 00000100:00000001:0.0:1787695784.312130:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298351929 00000100:00000001:1.0:1787695784.312131:0:2403:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695784.312139:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298344662 00000100:00000001:1.0:1787695784.312154:0:2403:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929565568 : -128555779986048 : ffff8b1446666d80) 00000100:00000001:0.0:1787695784.312167:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298351929 00000100:00000001:0.0:1787695784.312171:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1787695784.312175:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446666d80 x1874531139521280/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695869 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695784.312177:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695784.312187:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695784.312197:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695784.312201:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787695784.312203:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:1.0:1787695784.312205:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:0.0:1787695784.312207:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695784.312211:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:1.0:1787695784.312212:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46440. 00000100:00000001:0.0:1787695784.312213:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695784.312216:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695784.312219:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695784.312223:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874531139521280, offset 0 00000100:00000001:0.0:1787695784.312237:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1787695784.312238:0:2403:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:0.0:1787695784.312240:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695784.312242:0:2404:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695784.312250:0:2404:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446666300 x1874531139521536/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695784.312267:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695784.312279:0:2404:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695784.312286:0:2404:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695784.312287:0:2403:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 02000000:00000001:0.0:1787695784.312300:0:2404:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:1.0:1787695784.312304:0:2403:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:0.0:1787695784.312306:0:2404:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1787695784.312310:0:2403:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 02000000:00000001:0.0:1787695784.312314:0:2404:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695784.312319:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00100000:0.0:1787695784.312320:0:2404:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446666300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:608c526c-eb6e-4a54-80a5-8969828241eb:2404:1874531139521536:192.168.201.140@tcp:400:kworker.0 00000400:00000200:1.0:1787695784.312327:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:0.0:1787695784.312333:0:2404:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695784.312339:0:2404:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:1.0:1787695784.312339:0:2403:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16883:16883:254:4294967295] 192.168.201.140@tcp LPNI seq info [16882:16882:6:4294967295] 02000000:00000001:0.0:1787695784.312347:0:2404:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695784.312352:0:2404:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:1.0:1787695784.312353:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000010:0.0:1787695784.312388:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b1447624000. 02000000:00000001:0.0:1787695784.312393:0:2404:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695784.312404:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695784.312405:0:2403:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000400:00000010:0.0:1787695784.312410:0:2404:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bc60. 00000800:00000200:1.0:1787695784.312416:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695784.312418:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f967f8. 00000800:00000010:1.0:1787695784.312422:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1444663f00. 00000100:00000200:0.0:1787695784.312423:0:2404:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874531139521536, portal 10 00000100:00000001:0.0:1787695784.312428:0:2404:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695784.312430:0:2404:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929562880 : -128555779988736 : ffff8b1446666300) 00000800:00000200:1.0:1787695784.312436:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:0.0:1787695784.312442:0:2404:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446666300 x1874531139521536/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695869 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1F:1787695784.312448:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695784.312453:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695784.312462:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787695784.312464:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695784.312465:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 00000100:00000200:0.0:1787695784.312469:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874531139521536, offset 0 00000100:00000040:1.0:1787695784.312470:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:0.0:1787695784.312482:0:2404:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695784.312482:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695784.312488:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298344775 00000400:00000200:0.0:1787695784.312493:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695784.312497:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298351929 00000400:00000200:0.0:1787695784.312502:0:2404:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:1.0:1787695784.312502:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298344775 00000100:00000001:1.0:1787695784.312512:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298351929 00000400:00000200:0.0:1787695784.312514:0:2404:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1787695784.312519:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1787695784.312525:0:2404:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:1.0:1787695784.312526:0:2403:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695784.312532:0:2404:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1787695784.312534:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695784.312536:0:2404:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:1.0:1787695784.312539:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1787695784.312548:0:2404:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16884:16884:253:4294967295] 192.168.201.140@tcp LPNI seq info [16883:16883:5:4294967295] 00000100:00000001:1.0:1787695784.312548:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695784.312555:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=41 : 41 : 29) 00000400:00000200:0.0:1787695784.312561:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695784.312565:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695784.312572:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1787695784.312573:0:2404:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:1.0:1787695784.312579:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1787695784.312583:0:2404:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695784.312591:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da700. 00000100:00000001:1.0:1787695784.312591:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1787695784.312600:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695784.312605:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695784.312607:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695784.312610:0:2404:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:0.0:1787695784.312613:0:2404:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695784.312615:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298344641 00000100:00000001:0.0:1787695784.312618:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298351929 00000100:00000001:0.0:1787695784.312620:0:2404:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298344641 00000100:00000001:0.0:1787695784.312622:0:2404:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298351929 00000100:00000001:0.0:1787695784.312626:0:2404:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695784.312628:0:2404:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695784.312635:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695784.312637:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695784.312643:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695784.312645:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=31 : 31 : 1f) 00000100:00000001:0.0:1787695784.312648:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695784.312650:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695784.312652:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695784.312655:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695784.502439:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695784.502445:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695784.502450:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695784.502466:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695784.502469:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695784.502470:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695784.502473:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695784.502474:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695784.502476:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695784.502477:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695784.502480:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695784.502481:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695784.502535:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695784.502539:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695784.502543:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695784.886458:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695784.886464:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695784.886466:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695784.886470:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695784.886472:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695784.886475:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695784.886477:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695784.886479:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695784.886481:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.334426:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695785.334435:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1787695785.334449:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787695785.334452:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695765/real 1787695765] req@ffff8b1446605180 x1874531139520000/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695785 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695785.334460:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1787695785.334466:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695785.334474:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:1.0:1787695785.334477:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:0.0:1787695785.334479:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1787695785.334481:0:47074:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1787695785.334490:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665180. 00000020:00000040:1.0:1787695785.334494:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=26 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000400:00000001:0.0:1787695785.334497:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787695785.334501:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:0.0:1787695785.334503:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702058. 02000000:00000001:1.0:1787695785.334505:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:0.0:1787695785.334511:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 02000000:00000001:1.0:1787695785.334512:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695785.334516:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 02000000:00000010:1.0:1787695785.334521:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b1444663b00. 00000100:00000001:0.0:1787695785.334524:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695785.334530:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446605180 x1874531139520000/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695785 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695785.334530:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695785.334541:0:47074:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID req@ffff8b1446665180 x1874531139521664/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1787695785.334562:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446605180 x1874531139520000/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695785 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.334569:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695785.334571:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.334573:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695785.334574:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695785.334575:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695785.334582:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446605180 x1874531139520000/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695785 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695785.334582:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.334591:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.334595:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:1.0:1787695785.334595:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f923372e] to pc [ptlrpcd_00_00+0] req@ffff8b1446665180 x1874531139521664/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695785.334602:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695765/real 1787695765] req@ffff8b1446607800 x1874531139519872/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695785 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695785.334628:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695785.334630:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695785.334633:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702370. 00000400:00000200:0.0:1787695785.334637:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46000 00000400:00000010:0.0:1787695785.334639:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46000. 00000100:00000001:1.0:1787695785.334642:0:47074:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.334643:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:1.0:1787695785.334646:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:0.0:1787695785.334650:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446607800 x1874531139519872/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695785 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695785.334662:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1787695785.334667:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:0.0:1787695785.334671:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446607800 x1874531139519872/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695785 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695785.334673:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:1.0:1787695785.334679:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695785.334688:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000001:0.0:1787695785.334690:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:1.0:1787695785.334692:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695785.334694:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695785.334696:0:47074:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1787695785.334700:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446666680. 00000020:00000040:1.0:1787695785.334703:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=27 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000001:0.0:1787695785.334705:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695785.334708:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787695785.334709:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787695785.334710:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695785.334715:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695785.334719:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446607800 x1874531139519872/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695785 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1787695785.334722:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff8b1449817100. 00000100:00000001:1.0:1787695785.334729:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.334732:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695785.334738:0:47074:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID req@ffff8b1446666680 x1874531139521792/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695785.334743:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695785.334747:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695785.334749:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695785.334752:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695785.334754:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.334756:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695785.334762:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695785.334762:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bd4d567] to pc [ptlrpcd_00_01+1] req@ffff8b1446666680 x1874531139521792/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695785.334773:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446605180 x1874531139520000/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695785 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695785.334777:0:47074:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695785.334783:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695785.334790:0:2402:0:(client.c:2142:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:1.0:1787695785.334791:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:1.0:1787695785.334797:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695785.334798:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.334802:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:1.0:1787695785.334804:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1787695785.334806:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.334809:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695785.334814:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:1.0:1787695785.334817:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 4 (3683) 00000100:00080000:0.0:1787695785.334820:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1787695785.334833:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695785.334837:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695785.334839:0:2403:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1787695785.334845:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000040:1.0:1787695785.334847:0:2403:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665180 x1874531139521664/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.334850:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695785.334860:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446605180 x1874531139520000/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695785 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695785.334862:0:2403:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695785.334867:0:2403:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695785.334873:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446605180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139520000:192.168.201.140@tcp:8:kworker.0 02000000:00000001:1.0:1787695785.334874:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1787695785.334876:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1787695785.334879:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695785.334880:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787695785.334888:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695785.334888:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000040:0.0:1787695785.334891:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446607800 x1874531139519872/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695785 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695785.334895:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787695785.334897:0:2403:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2403:1874531139521664:192.168.201.140@tcp:400:kworker.0 00000100:00080000:2.0:1787695785.334906:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000001:1.0:1787695785.334910:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1787695785.334912:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1787695785.334918:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787695785.334921:0:2402:0:(client.c:2142:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695785.334921:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:1.0:1787695785.334923:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695785.334924:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695785.334927:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1787695785.334929:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787695785.334932:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787695785.334935:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1787695785.334936:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b1447532a00. 00000100:00080000:2.0:1787695785.334937:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3658 00000100:00000001:0.0:1787695785.334938:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695785.334941:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787695785.334943:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.334944:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695785.334948:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:2.0:1787695785.334948:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695785.334954:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:0.0:1787695785.334956:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000400:00000200:2.0:1787695785.334956:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000010:1.0:1787695785.334962:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547026e0. 00000100:00001000:2.0:1787695785.334963:0:47074:0:(import.c:2149:obd_at_measure()) add 20 to ffff8b1458083418 time=66 v=15 (15 1 1 1) 00000100:00001000:2.0:1787695785.334968:0:47074:0:(import.c:2215:obd_at_measure()) AT ffff8b1458083418 change: old=15 new=20 delta=5 (val=20) hist 20 1 1 1 00000400:00000010:1.0:1787695785.334971:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46550. 00000100:00080000:2.0:1787695785.334972:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 15s 00000100:00000040:2.0:1787695785.334978:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00080000:0.0:1787695785.334981:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000200:1.0:1787695785.334982:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874531139521664, portal 10 00000100:00000001:2.0:1787695785.334983:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695785.334988:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1787695785.334988:0:2403:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:2.0:1787695785.334990:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695785.334993:0:2403:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929558400 : -128555779993216 : ffff8b1446665180) 00000100:00000001:2.0:1787695785.334996:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000040:0.0:1787695785.334997:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446607800 x1874531139519872/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695785 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787695785.335004:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000040:1.0:1787695785.335006:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665180 x1874531139521664/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695870 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1787695785.335008:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:2.0:1787695785.335014:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:0.0:1787695785.335017:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446607800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139519872:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695785.335028:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695785.335029:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:2.0:1787695785.335029:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:2.0:1787695785.335035:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:0.0:1787695785.335036:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695785.335039:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000020:00000001:2.0:1787695785.335041:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:0.0:1787695785.335044:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446605180 x1874531139520000/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695785 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1787695785.335046:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000040:2.0:1787695785.335048:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000200:1.0:1787695785.335052:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874531139521664, offset 0 00000100:00000001:2.0:1787695785.335053:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695785.335058:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695785.335063:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695785.335068:0:2403:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000020:00000040:2.0:1787695785.335069:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00000001:0.0:1787695785.335072:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00080000:2.0:1787695785.335077:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695785.335080:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695785.335081:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:0.0:1787695785.335082:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787695785.335082:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:0.0:1787695785.335088:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:2.0:1787695785.335090:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1787695785.335094:0:2403:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 02000000:00000001:2.0:1787695785.335094:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695785.335097:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787695785.335098:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000020:00000001:0.0:1787695785.335103:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000008:00100000:2.0:1787695785.335103:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000020:00000040:0.0:1787695785.335108:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000400:00000200:1.0:1787695785.335109:0:2403:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:0.0:1787695785.335113:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000008:00000001:2.0:1787695785.335115:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695785.335117:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f7400. 00000400:00000200:1.0:1787695785.335118:0:2403:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 02000000:00000001:0.0:1787695785.335121:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1787695785.335122:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695785.335139:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ad880. 02000000:00000001:0.0:1787695785.335140:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695785.335143:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446605180. 00000020:00000040:2.0:1787695785.335144:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000400:00000200:1.0:1787695785.335145:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1787695785.335149:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 02000000:00000001:2.0:1787695785.335150:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787695785.335152:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1787695785.335153:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 02000000:00000001:2.0:1787695785.335154:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.335157:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787695785.335159:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695785.335162:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:0.0:1787695785.335164:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446607800 x1874531139519872/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695785 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695785.335165:0:2403:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16885:16885:252:4294967295] 192.168.201.140@tcp LPNI seq info [16884:16884:4:4294967295] 02000000:00000001:2.0:1787695785.335166:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695785.335174:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663800. 00000100:00000001:2.0:1787695785.335180:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695785.335182:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00100000:2.0:1787695785.335196:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ad880 x1874531139521920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695785.335203:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000400:00000200:1.0:1787695785.335205:0:2403:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 02000000:00000001:0.0:1787695785.335207:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695785.335210:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546000. 00000100:00000001:2.0:1787695785.335213:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1787695785.335216:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787695785.335217:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695785.335219:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000800:00000200:1.0:1787695785.335223:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000040:2.0:1787695785.335227:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000075b4d632] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ad880 x1874531139521920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1787695785.335229:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000800:00000010:1.0:1787695785.335232:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817e00. 00000020:00000001:0.0:1787695785.335237:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695785.335241:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f5000. 00000800:00000200:1.0:1787695785.335246:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:0.0:1787695785.335248:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695785.335254:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695785.335259:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446607800. 00000800:00000200:1.1:1787695785.335259:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:2.0:1787695785.335260:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.335267:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00080000:2.0:1787695785.335267:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695785.335268:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.335272:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787695785.335275:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1787695785.335278:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695785.335278:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000100:00000001:2.0:1787695785.335278:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695785.335282:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695785.335285:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695785.335286:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787695785.335289:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1787695785.335290:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:2.0:1787695785.335293:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3658 00000100:00000001:0.0:1787695785.335294:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695785.335297:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298345797 00000100:00000001:0.0:1787695785.335299:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695785.335302:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787695785.335304:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298351929 00000400:00000200:2.0:1787695785.335306:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:0.0:1787695785.335309:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ad880 x1874531139521920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695785.335312:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298345797 00000400:00000200:2.0:1787695785.335315:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695785.335319:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298351929 00000100:00001000:2.0:1787695785.335321:0:47074:0:(import.c:2149:obd_at_measure()) add 20 to ffff8b1458080c18 time=91 v=15 (15 0 0 0) 00000100:00000001:1.0:1787695785.335328:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695785.335330:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00001000:2.0:1787695785.335331:0:47074:0:(import.c:2215:obd_at_measure()) AT ffff8b1458080c18 change: old=15 new=20 delta=5 (val=20) hist 20 0 0 0 00000100:00000001:1.0:1787695785.335333:0:2403:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.335334:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695785.335337:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 15s 02000000:00000001:0.0:1787695785.335340:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1787695785.335343:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695785.335344:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695785.335345:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695785.335348:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695785.335349:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787695785.335350:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695785.335354:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695785.335356:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:0.0:1787695785.335358:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139521920:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787695785.335358:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695785.335360:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=40 : 40 : 28) 00000020:00000001:2.0:1787695785.335381:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695785.335383:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:2.0:1787695785.335385:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695785.335387:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000100:00000001:1.0:1787695785.335390:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695785.335391:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1787695785.335392:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000001:1.0:1787695785.335394:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787695785.335396:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:2.0:1787695785.335396:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000001:1.0:1787695785.335398:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695785.335399:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695785.335403:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695785.335405:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:1.0:1787695785.335406:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:2.0:1787695785.335406:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000010:0.0:1787695785.335410:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14475f9c00. 00000100:00000040:2.0:1787695785.335411:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695785.335412:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787695785.335415:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695785.335416:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695785.335419:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:2.0:1787695785.335420:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000400:00000200:0.0:1787695785.335423:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695785.335424:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787695785.335426:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695785.335429:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695785.335432:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000010:0.0:1787695785.335433:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b9a0. 00000100:00000001:2.0:1787695785.335435:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695785.335438:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:0.0:1787695785.335439:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96330. 02000000:00000001:2.0:1787695785.335441:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695785.335444:0:2403:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787695785.335445:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695785.335446:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139521920, portal 4 00000100:00000001:2.0:1787695785.335449:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787695785.335451:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695785.335454:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928806528 : -128555780745088 : ffff8b14465ad880) 00000100:00000040:1.0:1787695785.335454:0:2403:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446666680 x1874531139521792/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787695785.335455:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787695785.335461:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695785.335465:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695785.335467:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ad880 x1874531139521920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1787695785.335471:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ace00. 00000020:00000040:2.0:1787695785.335476:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 00000100:00000001:1.0:1787695785.335482:0:2403:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1787695785.335482:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695785.335486:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695785.335487:0:2403:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.335490:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787695785.335490:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695785.335494:0:2403:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695785.335494:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695785.335499:0:2403:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1787695785.335500:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 02000000:00000001:2.0:1787695785.335500:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695785.335506:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96440. 02000000:00000001:1.0:1787695785.335507:0:2403:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695785.335510:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661c00. 00000100:00000200:0.0:1787695785.335514:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139521920, offset 0 00000100:00100000:1.0:1787695785.335517:0:2403:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446666680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:608c526c-eb6e-4a54-80a5-8969828241eb:2403:1874531139521792:192.168.201.140@tcp:400:kworker.0 00000100:00000001:2.0:1787695785.335517:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695785.335526:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787695785.335527:0:2403:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:2.0:1787695785.335530:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ace00 x1874531139522048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787695785.335533:0:2403:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695785.335538:0:2403:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695785.335539:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:1.0:1787695785.335544:0:2403:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:0.0:1787695785.335550:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 02000000:00000010:1.0:1787695785.335551:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff8b1447533800. 00000100:00000001:2.0:1787695785.335552:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787695785.335557:0:2403:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695785.335558:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1787695785.335558:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695785.335566:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695785.335566:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:2.0:1787695785.335571:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046eddc29] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ace00 x1874531139522048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695785.335573:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000010:1.0:1787695785.335574:0:2403:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702e70. 00000400:00000200:0.0:1787695785.335580:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000010:1.0:1787695785.335583:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46bb0. 00000400:00000200:0.0:1787695785.335584:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16886:16886:251:4294967295] 192.168.201.140@tcp LPNI seq info [16885:16885:3:4294967295] 00000100:00000200:1.0:1787695785.335590:0:2403:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1874531139521792, portal 10 00000400:00000200:0.0:1787695785.335594:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695785.335597:0:2403:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695785.335601:0:2403:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929563776 : -128555779987840 : ffff8b1446666680) 00000100:00000001:2.0:1787695785.335607:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695785.335610:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00080000:2.0:1787695785.335612:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1787695785.335615:0:2403:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446666680 x1874531139521792/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695870 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695785.335621:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000800:00000200:0.0:1787695785.335626:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000040:2.0:1787695785.335628:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:0.0:1787695785.335634:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da800. 00000100:00000040:2.0:1787695785.335640:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1787695785.335641:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000200:0.0:1787695785.335646:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:2.0:1787695785.335647:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1787695785.335651:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695785.335655:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46660. 00000800:00000200:0.1:1787695785.335657:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000200:1.0:1787695785.335661:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1874531139521792, offset 0 00000100:00000040:2.0:1787695785.335664:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1787695785.335665:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695785.335672:0:2403:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:0.0:1787695785.335673:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000100:00000040:2.0:1787695785.335673:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 4 (3683) 00000100:00080000:2.0:1787695785.335679:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695785.335681:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695785.335683:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:2.0:1787695785.335687:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000001:0.0:1787695785.335689:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298345828 00000100:00080000:2.0:1787695785.335691:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695785.335692:0:2403:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:0.0:1787695785.335696:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298351929 00000100:00080000:2.0:1787695785.335700:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000001:0.0:1787695785.335701:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298345828 00000400:00000200:1.0:1787695785.335703:0:2403:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:2.0:1787695785.335705:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695785.335708:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000400:00000200:1.0:1787695785.335710:0:2403:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:0.0:1787695785.335713:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298351929 00000100:00000040:2.0:1787695785.335714:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695785.335716:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1787695785.335718:0:2403:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1787695785.335722:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695785.335723:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1787695785.335727:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695785.335728:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695785.335731:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1787695785.335733:0:2403:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000040:2.0:1787695785.335735:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1787695785.335738:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:2.0:1787695785.335740:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 4 (3683) 00000100:00000001:0.0:1787695785.335741:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1787695785.335743:0:2403:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16887:16887:250:4294967295] 192.168.201.140@tcp LPNI seq info [16886:16886:2:4294967295] 00000100:00000001:0.0:1787695785.335746:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695785.335748:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695785.335751:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000400:00000200:1.0:1787695785.335753:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:0.0:1787695785.335759:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ace00 x1874531139522048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695785.335772:0:2403:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:0.0:1787695785.335774:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695785.335778:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695785.335783:0:2403:0:(socklnd_cb.c:931:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.201.140@tcp 02000000:00000001:0.0:1787695785.335784:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695785.335788:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000800:00000010:1.0:1787695785.335789:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817700. 02000000:00000001:0.0:1787695785.335791:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695785.335799:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00100000:0.0:1787695785.335800:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139522048:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695785.335809:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000800:00000200:1.1:1787695785.335810:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:0.0:1787695785.335813:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695785.335815:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695785.335815:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695785.335819:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1787695785.335823:0:2403:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 02000000:00000010:0.0:1787695785.335824:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14475f8800. 02000000:00000001:0.0:1787695785.335830:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695785.335834:0:2403:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695785.335837:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695785.335839:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298345664 00000400:00000010:0.0:1787695785.335841:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b6e0. 00000100:00000001:1.0:1787695785.335845:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298351929 00000400:00000010:0.0:1787695785.335847:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96550. 00000100:00000001:1.0:1787695785.335851:0:2403:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298345664 00000100:00000200:0.0:1787695785.335853:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139522048, portal 4 00000100:00000001:0.0:1787695785.335857:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695785.335857:0:2403:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298351929 00000100:00000001:0.0:1787695785.335859:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928803840 : -128555780747776 : ffff8b14465ace00) 00000100:00000001:1.0:1787695785.335863:0:2403:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1787695785.335868:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ace00 x1874531139522048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695785.335869:0:2403:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695785.335876:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.335877:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787695785.335881:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695785.335887:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695785.335891:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96220. 00000100:00000200:0.0:1787695785.335895:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139522048, offset 0 00000400:00000200:0.0:1787695785.335900:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695785.335905:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695785.335907:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1787695785.335912:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:1.0:1787695785.335912:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1787695785.335919:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1787695785.335919:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695785.335924:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:1.0:1787695785.335924:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695785.335930:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695785.335935:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695785.335938:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16888:16888:249:4294967295] 192.168.201.140@tcp LPNI seq info [16887:16887:1:4294967295] 00000400:00000200:0.0:1787695785.335941:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695785.335949:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695785.335953:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695785.335955:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:0.0:1787695785.335958:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695785.335961:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695785.335963:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695785.335966:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:0.0:1787695785.335968:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695785.335969:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298345500 00000100:00000001:0.0:1787695785.335971:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298351929 00000100:00000001:0.0:1787695785.335973:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298345500 00000100:00000001:0.0:1787695785.335974:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298351929 00000100:00000001:0.0:1787695785.335975:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695785.335977:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.335979:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.335980:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.335982:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695785.335983:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.335985:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695785.335986:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695785.335992:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.335993:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695785.526437:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695785.526443:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695785.526447:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695785.526448:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695785.526451:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695785.526452:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695785.526455:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695785.526456:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695785.526458:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695785.526459:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695785.526476:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695785.526477:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695785.526540:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695785.526545:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695785.526554:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787695785.654619:0:2392:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.201.140:988 00000400:00000100:2.0:1787695785.654660:0:2392:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.140@tcp at host 192.168.201.140 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1787695785.654670:0:2392:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695785.654674:0:2392:0:(router.c:1791:lnet_notify()) 192.168.201.40@tcp notifying 192.168.201.140@tcp: down 00000400:00000200:2.0:1787695785.654679:0:2392:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695785.654684:0:2392:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:2.0:1787695785.654688:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:2.0:1787695785.654692:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da500. 00000400:00000200:2.0:1787695785.654697:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1787695785.654702:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1787695785.654710:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96660 00000400:00000010:2.0:1787695785.654726:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96660. 00000100:00000001:2.0:1787695785.654732:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695785.654740:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b1446667800 x1874531139521152/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695809 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695785.654766:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695785.654770:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446667800 x1874531139521152/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695809 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695785.654780:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695785.654782:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787695785.654794:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:2.0:1787695785.654797:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460daa00. 00000400:00000200:2.0:1787695785.654801:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1787695785.654806:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1787695785.654811:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f962a8 00000400:00000010:2.0:1787695785.654814:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f962a8. 00000100:00000001:0.0:1787695785.654815:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695785.654818:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787695785.654823:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695785.654830:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000200:2.0:1787695785.654831:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b1446664000 x1874531139521408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695809 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695785.654841:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695784/real 1787695785] req@ffff8b1446667800 x1874531139521152/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695785.654847:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695785.654855:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446664000 x1874531139521408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695809 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695785.654863:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695785.654868:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787695785.654868:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695785.654873:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000010:0.0:1787695785.654877:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b8f0. 00000800:00000100:2.0:1787695785.654884:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 224 0@<0:0>->192.168.201.140@tcp 00000400:00000200:0.0:1787695785.654885:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96198 00000400:00000010:0.0:1787695785.654891:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96198. 00000800:00000010:2.0:1787695785.654891:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1444663f00. 00000100:00000001:0.0:1787695785.654900:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:2.0:1787695785.654900:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000200:0.0:1787695785.654908:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446667800 x1874531139521152/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787695785.654909:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1787695785.654916:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46440 00000400:00000010:2.0:1787695785.654919:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46440. 00000100:00000001:2.0:1787695785.654927:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695785.654932:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446667800 x1874531139521152/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695785.654936:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b1446666d80 x1874531139521280/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695869 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.654951:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695785.654954:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.654958:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695785.654962:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695785.654973:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446667800 x1874531139521152/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.654981:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695785.654987:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446667800 x1874531139521152/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695809 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695785.654994:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787695785.654996:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695785.655002:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695785.655003:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446666d80 x1874531139521280/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695869 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695785.655012:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.655015:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695785.655016:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695785.655020:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.655025:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000800:00000100:2.0:1787695785.655028:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 224 0@<0:0>->192.168.201.140@tcp 00000100:00000001:0.0:1787695785.655030:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000800:00000010:2.0:1787695785.655033:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da700. 00000100:00080000:0.0:1787695785.655037:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000400:00000200:2.0:1787695785.655039:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00080000:0.0:1787695785.655050:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000400:00000200:2.0:1787695785.655050:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000100:00000001:0.0:1787695785.655059:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000400:00000200:2.0:1787695785.655060:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:2.0:1787695785.655064:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:2.0:1787695785.655069:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:0.0:1787695785.655070:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446667800 x1874531139521152/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695809 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695785.655077:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b1446666300 x1874531139521536/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695869 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695785.655092:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446667800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139521152:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695785.655099:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787695785.655101:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.655107:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695785.655107:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446666300 x1874531139521536/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695869 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695785.655114:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446667800 x1874531139521152/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695809 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695785.655135:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695785.655139:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695785.655143:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695785.655149:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000800:00000100:2.0:1787695785.655149:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 224 0@<0:0>->192.168.201.140@tcp 00000800:00000010:2.0:1787695785.655154:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817e00. 02000000:00000010:0.0:1787695785.655155:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14475fbc00. 00000400:00000200:2.0:1787695785.655159:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 02000000:00000001:0.0:1787695785.655163:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:2.0:1787695785.655167:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000020:00000001:0.0:1787695785.655168:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000400:00000200:2.0:1787695785.655172:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000020:00000040:0.0:1787695785.655173:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000400:00000010:2.0:1787695785.655177:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000020:00000001:0.0:1787695785.655178:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:2.0:1787695785.655182:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695785.655189:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b1446665180 x1874531139521664/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695870 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1787695785.655190:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446544c00. 02000000:00000001:0.0:1787695785.655199:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1787695785.655201:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787695785.655204:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:2.0:1787695785.655208:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665180 x1874531139521664/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695870 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1787695785.655209:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446667800. 00000100:00000001:0.0:1787695785.655215:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695785.655217:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.655223:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.655228:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695785.655232:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695785.655232:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.655235:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787695785.655237:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:0.0:1787695785.655244:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695784/real 1787695785] req@ffff8b1446664000 x1874531139521408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1787695785.655248:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:2.0:1787695785.655251:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da800. 00000400:00000200:2.0:1787695785.655257:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1787695785.655267:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1787695785.655273:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96440 00000400:00000001:0.0:1787695785.655276:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000010:2.0:1787695785.655278:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96440. 00000400:00000001:0.0:1787695785.655280:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787695785.655283:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000400:00000010:0.0:1787695785.655284:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b7e8. 00000400:00000200:0.0:1787695785.655291:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96b28 00000100:00000200:2.0:1787695785.655293:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14465ad880 x1874531139521920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1787695785.655295:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96b28. 00000100:00000001:0.0:1787695785.655301:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695785.655309:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446664000 x1874531139521408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695785.655311:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695785.655319:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ad880 x1874531139521920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695810 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695785.655326:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446664000 x1874531139521408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695785.655333:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695785.655338:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695785.655342:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695785.655346:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.655350:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000800:00000100:2.0:1787695785.655351:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 224 0@<0:0>->192.168.201.140@tcp 00000100:00000001:0.0:1787695785.655353:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:2.0:1787695785.655356:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817700. 00000100:00100000:0.0:1787695785.655362:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446664000 x1874531139521408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.655401:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1787695785.655405:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1787695785.655412:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000100:00000040:0.0:1787695785.655413:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446664000 x1874531139521408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695809 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787695785.655417:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46660 00000400:00000010:2.0:1787695785.655421:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46660. 00000100:00000001:2.0:1787695785.655427:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787695785.655432:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000200:2.0:1787695785.655434:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b1446666680 x1874531139521792/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695870 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.655436:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.655442:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695785.655445:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.655449:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695785.655454:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695785.655455:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1787695785.655459:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000040:2.0:1787695785.655462:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446666680 x1874531139521792/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 0 dl 1787695870 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787695785.655467:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695785.655473:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1787695785.655477:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695785.655480:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1787695785.655488:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446664000 x1874531139521408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695809 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695785.655500:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446664000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139521408:192.168.201.140@tcp:38:kworker.0 00000800:00000100:2.0:1787695785.655503:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:2.0:1787695785.655510:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000100:00000001:0.0:1787695785.655511:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695785.655515:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:0.0:1787695785.655516:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695785.655523:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446664000 x1874531139521408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695809 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787695785.655524:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1787695785.655531:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96220 00000400:00000010:2.0:1787695785.655535:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96220. 00000100:00000001:0.0:1787695785.655540:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1787695785.655542:0:2392:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1787695785.655544:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695785.655549:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14475f9800. 00000100:00000200:2.0:1787695785.655550:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14465ace00 x1874531139522048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787695785.655555:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695785.655559:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695785.655562:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000100:00000001:2.0:1787695785.655566:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1787695785.655567:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695785.655570:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545000. 00000100:00000040:2.0:1787695785.655573:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ace00 x1874531139522048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695810 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787695785.655576:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695785.655580:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695785.655583:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446664000. 00000100:00000001:2.0:1787695785.655586:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.655589:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695785.655589:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695785.655594:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.655598:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.655600:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695785.655602:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695785.655604:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695785.655610:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695785/real 1787695785] req@ffff8b14465ace00 x1874531139522048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695785.655621:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695785.655623:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695785.655625:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b6e0. 00000400:00000200:0.0:1787695785.655629:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96550 00000400:00000010:0.0:1787695785.655631:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96550. 00000100:00000001:0.0:1787695785.655636:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695785.655648:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ace00 x1874531139522048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695785.655658:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ace00 x1874531139522048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.655665:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695785.655667:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.655669:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695785.655671:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695785.655674:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ace00 x1874531139522048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.655680:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695785.655684:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ace00 x1874531139522048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.655690:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695785.655693:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.655694:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695785.655695:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.655697:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695785.655699:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695785.655702:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695785.655705:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695785.655707:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695785.655711:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ace00 x1874531139522048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695785.655718:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139522048:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695785.655722:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695785.655726:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695785/real 1787695785] req@ffff8b14465ad880 x1874531139521920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695785.655744:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695785.655745:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695785.655747:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b9a0. 00000400:00000200:0.0:1787695785.655750:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96330 00000400:00000010:0.0:1787695785.655752:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96330. 00000100:00000001:0.0:1787695785.655754:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695785.655758:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ad880 x1874531139521920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695785.655765:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ad880 x1874531139521920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.655772:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695785.655773:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.655775:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695785.655776:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695785.655780:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ad880 x1874531139521920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.655786:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695785.655789:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ad880 x1874531139521920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.655795:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695785.655797:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.655799:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695785.655800:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.655802:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695785.655804:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695785.655807:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695785.655810:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695785.655812:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695785.655821:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ad880 x1874531139521920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695785.655828:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139521920:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695785.655832:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.655834:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695785.655837:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ace00 x1874531139522048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.655842:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695785.655844:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695785.655845:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14475f8800. 02000000:00000001:0.0:1787695785.655851:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695785.655852:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695785.655853:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:0.0:1787695785.655855:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695785.655857:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661c00. 02000000:00000001:0.0:1787695785.655861:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695785.655862:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695785.655863:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ace00. 00000100:00000001:0.0:1787695785.655867:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695785.655869:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.655870:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695785.655873:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ad880 x1874531139521920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.655879:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695785.655880:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695785.655881:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14475f9c00. 02000000:00000001:0.0:1787695785.655884:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695785.655885:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695785.655886:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:0.0:1787695785.655888:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695785.655889:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663800. 02000000:00000001:0.0:1787695785.655891:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695785.655901:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695785.655903:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ad880. 00000100:00000001:0.0:1787695785.655905:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695785.655907:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.655909:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.655911:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695785.655912:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695785.655914:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.655916:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695785.655933:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695785.655937:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695785.655939:0:2404:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000400:1.0:1787695785.655947:0:2404:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695784/real 1787695785] req@ffff8b1446666300 x1874531139521536/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.655948:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695785.655953:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695785.655956:0:2403:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000400:00000001:1.0:1787695785.655964:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000400:0.0:1787695785.655966:0:2403:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695785/real 1787695785] req@ffff8b1446666680 x1874531139521792/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695785.655967:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695785.655971:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bc60. 00000400:00000200:1.0:1787695785.655977:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f967f8 00000400:00000010:1.0:1787695785.655982:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f967f8. 00000100:00000001:1.0:1787695785.655986:0:2404:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000001:0.0:1787695785.655987:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695785.655991:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000200:1.0:1787695785.655992:0:2404:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446666300 x1874531139521536/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1787695785.655995:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702e70. 00000400:00000200:0.0:1787695785.656001:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46bb0 00000400:00000010:0.0:1787695785.656005:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46bb0. 00000100:00000200:1.0:1787695785.656007:0:2404:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446666300 x1874531139521536/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.656011:0:2403:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787695785.656020:0:2404:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000200:0.0:1787695785.656021:0:2403:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446666680 x1874531139521792/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695785.656023:0:2404:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695785.656029:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695785.656032:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695785.656042:0:2404:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446666300 x1874531139521536/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695785.656043:0:2403:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446666680 x1874531139521792/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695785.656068:0:2404:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00000001:0.0:1787695785.656070:0:2403:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695785.656074:0:2403:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787695785.656074:0:2404:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from FULL to DISCONN 00000100:02000400:1.0:1787695785.656082:0:2404:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0001-mdc-ffff8b145807d000: Connection to lustre-MDT0001 (at 192.168.201.140@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00000001:0.0:1787695785.656086:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695785.656089:0:2404:0:(obd_class.h:1222:obd_import_event()) Process entered 00000100:00000001:0.0:1787695785.656090:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:1.0:1787695785.656094:0:2404:0:(mdc_request.c:2879:mdc_import_event()) Process entered 00000100:00100000:0.0:1787695785.656099:0:2403:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446666680 x1874531139521792/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000002:00000001:1.0:1787695785.656100:0:2404:0:(mdc_request.c:2953:mdc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695785.656106:0:2404:0:(obd_class.h:1233:obd_import_event()) Process leaving 00000100:00080000:1.0:1787695785.656110:0:2404:0:(import.c:418:ptlrpc_pinger_force()) lustre-MDT0001_UUID: waking up pinger s:DISCONN 00000100:00000001:0.0:1787695785.656113:0:2403:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:0.0:1787695785.656117:0:2403:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b144515c000 already not connected (conn 11, was 11): DISCONN 00000100:00000001:0.0:1787695785.656131:0:2403:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:0.0:1787695785.656138:0:2403:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695785.656142:0:2403:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446666680 x1874531139521792/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695870 ref 1 fl Rpc:eXNQU/200/ffffffff rc -5/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.656148:0:2403:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551611 : -5 : 0xfffffffffffffffb) 00000100:00000001:1.0:1787695785.656148:0:2404:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:0.0:1787695785.656150:0:2403:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.656152:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695785.656153:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695785.656155:0:2403:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446666680 x1874531139521792/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695870 ref 1 fl Interpret:eXNQU/200/ffffffff rc -5/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695785.656160:0:2404:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695785.656163:0:2403:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446666680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:608c526c-eb6e-4a54-80a5-8969828241eb:2403:1874531139521792:192.168.201.140@tcp:400:kworker.0 00000100:00000001:0.0:1787695785.656169:0:2403:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:1.0:1787695785.656173:0:2404:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446666300 x1874531139521536/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695869 ref 1 fl Rpc:eXNQU/200/ffffffff rc -5/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:0.0:1787695785.656175:0:2403:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695785/real 1787695785] req@ffff8b1446665180 x1874531139521664/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695785.656202:0:2404:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551611 : -5 : 0xfffffffffffffffb) 00000400:00000001:0.0:1787695785.656208:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1787695785.656209:0:2404:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:0.0:1787695785.656213:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787695785.656214:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000010:0.0:1787695785.656219:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547026e0. 00000100:00000001:1.0:1787695785.656219:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1787695785.656226:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46550 00000400:00000010:0.0:1787695785.656231:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46550. 00000100:00000040:1.0:1787695785.656231:0:2404:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446666300 x1874531139521536/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695869 ref 1 fl Interpret:eXNQU/200/ffffffff rc -5/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.656237:0:2403:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695785.656245:0:2403:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665180 x1874531139521664/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695785.656250:0:2404:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446666300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:608c526c-eb6e-4a54-80a5-8969828241eb:2404:1874531139521536:192.168.201.140@tcp:400:kworker.0 00000100:00000200:0.0:1787695785.656265:0:2403:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665180 x1874531139521664/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695785.656265:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695785.656270:0:2404:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787695785.656278:0:2403:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:1.0:1787695785.656279:0:2404:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446666300 x1874531139521536/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695869 ref 1 fl Complete:eXNQU/200/ffffffff rc -5/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.656281:0:2403:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.656286:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695785.656290:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695785.656298:0:2404:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00100000:0.0:1787695785.656300:0:2403:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446665180 x1874531139521664/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695785.656301:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695785.656305:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1447624000. 02000000:00000001:1.0:1787695785.656313:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695785.656315:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695785.656318:0:2404:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=26 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695785.656324:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695785.656327:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1444663c00. 00000100:00000001:0.0:1787695785.656334:0:2403:0:(import.c:429:ptlrpc_fail_import()) Process entered 02000000:00000001:1.0:1787695785.656335:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:0.0:1787695785.656339:0:2403:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from FULL to DISCONN 02000000:00000001:1.0:1787695785.656339:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695785.656341:0:2404:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446666300. 00000100:00000001:1.0:1787695785.656348:0:2404:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:02000400:0.0:1787695785.656349:0:2403:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0001-mdc-ffff8b1458183800: Connection to lustre-MDT0001 (at 192.168.201.140@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00000001:1.0:1787695785.656352:0:2404:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.656354:0:2403:0:(obd_class.h:1222:obd_import_event()) Process entered 00000100:00000001:1.0:1787695785.656357:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1787695785.656358:0:2403:0:(mdc_request.c:2879:mdc_import_event()) Process entered 00000002:00000001:0.0:1787695785.656392:0:2403:0:(mdc_request.c:2953:mdc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.656396:0:2403:0:(obd_class.h:1233:obd_import_event()) Process leaving 00000100:00080000:0.0:1787695785.656398:0:2403:0:(import.c:418:ptlrpc_pinger_force()) lustre-MDT0001_UUID: waking up pinger s:DISCONN 00000100:00000001:1.0:1787695785.656401:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695785.656403:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695785.656406:0:2403:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:1.0:1787695785.656406:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.656410:0:2403:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695785.656410:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695785.656419:0:2403:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665180 x1874531139521664/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695870 ref 1 fl Rpc:eXNQU/200/ffffffff rc -5/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.656430:0:2403:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551611 : -5 : 0xfffffffffffffffb) 00000100:00000001:0.0:1787695785.656434:0:2403:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.656437:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695785.656439:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695785.656446:0:2403:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665180 x1874531139521664/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695870 ref 1 fl Interpret:eXNQU/200/ffffffff rc -5/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695785.656453:0:2403:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2403:1874531139521664:192.168.201.140@tcp:400:kworker.0 00000100:00000001:0.0:1787695785.656457:0:2403:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000400:0.0:1787695785.656460:0:2403:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695784/real 1787695785] req@ffff8b1446666d80 x1874531139521280/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695785.656467:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695785.656468:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695785.656470:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702318. 00000400:00000200:0.0:1787695785.656474:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46220 00000400:00000010:0.0:1787695785.656476:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46220. 00000100:00000001:0.0:1787695785.656483:0:2403:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695785.656488:0:2403:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446666d80 x1874531139521280/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695785.656494:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:0.0:1787695785.656496:0:2403:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446666d80 x1874531139521280/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695785.656517:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000001:0.0:1787695785.656522:0:2403:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:2.0:1787695785.656525:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695785.656526:0:2403:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.656537:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695785.656538:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695785.656541:0:2403:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446666d80 x1874531139521280/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695785.656546:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787695785.656550:0:2403:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:0.0:1787695785.656553:0:2403:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b1458081800 already not connected (conn 15, was 15): DISCONN 00000100:00000001:2.0:1787695785.656554:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787695785.656559:0:2403:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:0.0:1787695785.656562:0:2403:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787695785.656563:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:0.0:1787695785.656567:0:2403:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446666d80 x1874531139521280/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695869 ref 1 fl Rpc:eXNQU/200/ffffffff rc -5/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695785.656578:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787695785.656580:0:2403:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551611 : -5 : 0xfffffffffffffffb) 00000100:00000001:0.0:1787695785.656585:0:2403:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.656586:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695785.656587:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695785.656593:0:2403:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446666d80 x1874531139521280/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695869 ref 1 fl Interpret:eXNQU/200/ffffffff rc -5/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695785.656593:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3677 00000100:00100000:0.0:1787695785.656610:0:2403:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446666d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2403:1874531139521280:192.168.201.140@tcp:400:kworker.0 00000100:00000001:0.0:1787695785.656620:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.656622:0:2403:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695785.656624:0:2403:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446666680 x1874531139521792/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695870 ref 1 fl Complete:eXNQU/200/ffffffff rc -5/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787695785.656625:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695785.656631:0:2403:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695785.656637:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695785.656638:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1447533800. 00000400:00000200:2.0:1787695785.656643:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 02000000:00000001:0.0:1787695785.656644:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695785.656649:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695785.656653:0:2403:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=25 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695785.656657:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00080000:2.0:1787695785.656657:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 02000000:00000010:0.0:1787695785.656659:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1449817100. 02000000:00000001:0.0:1787695785.656670:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695785.656672:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695785.656675:0:2403:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446666680. 00000100:00000040:2.0:1787695785.656675:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695785.656683:0:2403:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695785.656686:0:2403:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695785.656687:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695785.656688:0:2403:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695785.656697:0:2403:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665180 x1874531139521664/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695870 ref 1 fl Complete:eXNQU/200/ffffffff rc -5/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695785.656697:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695785.656709:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695785.656712:0:2403:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695785.656713:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695785.656714:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1447532a00. 02000000:00000001:0.0:1787695785.656716:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695785.656717:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695785.656718:0:2403:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=25 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695785.656719:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695785.656720:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1444663b00. 02000000:00000001:0.0:1787695785.656721:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695785.656722:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695785.656723:0:2403:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665180. 00000100:00000001:0.0:1787695785.656724:0:2403:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695785.656725:0:2403:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.656726:0:2403:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695785.656730:0:2403:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446666d80 x1874531139521280/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695869 ref 1 fl Complete:eXNQU/200/ffffffff rc -5/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787695785.656736:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787695785.656738:0:2403:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695785.656740:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000040:2.0:1787695785.656741:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 02000000:00000010:0.0:1787695785.656742:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1447532e00. 00000020:00000001:2.0:1787695785.656750:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1787695785.656751:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695785.656753:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695785.656758:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000040:0.0:1787695785.656760:0:2403:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=24 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695785.656772:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 00000020:00000001:2.0:1787695785.656772:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 02000000:00000010:0.0:1787695785.656776:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1445678300. 02000000:00000001:0.0:1787695785.656783:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:2.0:1787695785.656784:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 02000000:00000001:0.0:1787695785.656787:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695785.656789:0:2403:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446666d80. 00000100:00000001:0.0:1787695785.656797:0:2403:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000040:2.0:1787695785.656802:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695785.656803:0:2403:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695785.656810:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695785.656813:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695785.656817:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695785.656819:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695785.656825:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695785.656826:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695785.656831:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1787695785.656833:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00000001:0.0:1787695785.656836:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695785.656842:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695785.656847:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695785.656850:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695785.656859:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695785.656862:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695785.656865:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695785.656870:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787695785.656873:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695785.656876:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695785.656882:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ae680. 00000020:00000040:2.0:1787695785.656886:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:2.0:1787695785.656890:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695785.656892:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695785.656896:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695785.656899:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695785.656904:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695785.656912:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445662c00. 00000100:00000001:2.0:1787695785.656931:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695785.656942:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ae680 x1874531139522176/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695785.656952:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695785.656964:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695785.656971:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019831621] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ae680 x1874531139522176/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695785.656994:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695785.656997:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695785.657001:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787695785.657002:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695785.657004:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695785.657007:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695785.657011:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3576 00000400:00000200:2.0:1787695785.657018:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695785.657024:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:0.0:1787695785.657025:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695785.657030:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1787695785.657032:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695785.657033:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787695785.657040:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695785.657042:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ae680 x1874531139522176/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695785.657049:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695785.657059:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695785.657060:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695785.657063:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787695785.657069:0:47074:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:0.0:1787695785.657070:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695785.657076:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1787695785.657076:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:2.0:1787695785.657084:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1787695785.657085:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1787695785.657093:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000100:00100000:0.0:1787695785.657094:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139522176:192.168.201.140@tcp:38:kworker.0 00000020:00000001:2.0:1787695785.657105:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000001:0.0:1787695785.657106:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695785.657109:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695785.657115:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787695785.657115:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 02000000:00000001:0.0:1787695785.657138:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:2.0:1787695785.657143:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000010:0.0:1787695785.657144:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14475f9c00. 02000000:00000001:0.0:1787695785.657153:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695785.657155:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695785.657161:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695785.657162:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695785.657171:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b9a0. 00000100:00000001:2.0:1787695785.657172:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000010:0.0:1787695785.657176:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96330. 00000020:00000040:2.0:1787695785.657178:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00000200:0.0:1787695785.657183:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139522176, portal 10 00000100:00080000:2.0:1787695785.657187:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695785.657188:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695785.657193:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.657195:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928810112 : -128555780741504 : ffff8b14465ae680) 00000100:00000040:0.0:1787695785.657204:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ae680 x1874531139522176/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695785.657206:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695785.657215:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695785.657219:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1787695785.657225:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695785.657228:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695785.657234:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:0.0:1787695785.657235:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96550. 00000100:00000200:0.0:1787695785.657240:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139522176, offset 0 00000400:00000200:0.0:1787695785.657244:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000008:00100000:2.0:1787695785.657247:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1787695785.657258:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000008:00000001:2.0:1787695785.657261:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695785.657266:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:2.0:1787695785.657267:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695785.657276:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000010:2.0:1787695785.657280:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ac700. 00000400:00000200:0.0:1787695785.657282:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000020:00000040:2.0:1787695785.657289:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=25 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000400:00000200:0.0:1787695785.657292:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695785.657299:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 02000000:00000001:2.0:1787695785.657300:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695785.657305:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695785.657306:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16889:16889:256:4294967295] 192.168.201.140@tcp LPNI seq info [16888:16888:8:4294967295] 00000400:00000200:0.0:1787695785.657318:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695785.657320:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695785.657324:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695785.657328:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695785.657339:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 02000000:00000010:2.0:1787695785.657340:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663c00. 00000100:00000001:2.0:1787695785.657350:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1787695785.657357:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695785.657392:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:0.0:1787695785.657401:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00100000:2.0:1787695785.657401:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ac700 x1874531139522304/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.0:1787695785.657410:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 1, 1 secs later) 00000800:00000200:0.1:1787695785.657426:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695785.657431:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 1, 1 secs later) 00000100:00000001:2.0:1787695785.657434:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000100:0.0:1787695785.657439:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695785.657442:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1787695785.657443:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:0.0:1787695785.657451:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:2.0:1787695785.657458:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007c6208f8] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ac700 x1874531139522304/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695785.657463:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695785.657469:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96550 00000400:00000010:0.0:1787695785.657474:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96550. 00000100:00000001:0.0:1787695785.657481:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695785.657490:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ae680 x1874531139522176/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695785.657499:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.657505:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1787695785.657508:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1787695785.657518:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ae680 x1874531139522176/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695810 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695785.657518:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787695785.657523:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695785.657530:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695785.657539:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695785.657542:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787695785.657543:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695785.657552:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695785.657552:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3677 00000100:00000040:0.0:1787695785.657561:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000400:00000200:2.0:1787695785.657569:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695785.657574:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695785.657580:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298347272 00000400:00000200:2.0:1787695785.657584:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:0.0:1787695785.657586:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298351929 00000100:00000001:0.0:1787695785.657591:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298347272 00000100:00080000:2.0:1787695785.657591:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787695785.657594:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298351929 00000100:00000001:0.0:1787695785.657597:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695785.657599:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.657601:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695785.657606:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695785/real 1787695785] req@ffff8b14465ae680 x1874531139522176/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695785.657613:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695785.657625:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:0.0:1787695785.657626:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695785.657631:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695785.657635:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b9a0. 00000100:00000040:2.0:1787695785.657636:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695785.657641:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96330 00000400:00000010:0.0:1787695785.657644:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96330. 00000100:00000001:2.0:1787695785.657645:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695785.657647:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695785.657657:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ae680 x1874531139522176/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787695785.657657:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695785.657664:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:2.0:1787695785.657673:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000200:0.0:1787695785.657676:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ae680 x1874531139522176/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1787695785.657680:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000020:00000001:2.0:1787695785.657695:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:0.0:1787695785.657696:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695785.657699:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.657707:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1787695785.657708:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:0.0:1787695785.657709:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695785.657719:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00100000:0.0:1787695785.657720:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ae680 x1874531139522176/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695785.657728:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695785.657738:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695785.657740:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000040:0.0:1787695785.657744:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ae680 x1874531139522176/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695785.657747:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695785.657757:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00000001:0.0:1787695785.657769:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695785.657771:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695785.657779:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695785.657779:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695785.657788:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787695785.657789:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695785.657793:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787695785.657795:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695785.657796:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1787695785.657803:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.657804:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695785.657806:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695785.657809:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787695785.657815:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695785.657817:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695785.657818:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000010:2.0:1787695785.657821:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465af480. 00000020:00000040:2.0:1787695785.657825:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:2.0:1787695785.657828:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00080000:0.0:1787695785.657830:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 02000000:00000001:2.0:1787695785.657831:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695785.657835:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695785.657839:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695785.657841:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.657842:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000010:2.0:1787695785.657847:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445662400. 00000100:00000040:0.0:1787695785.657851:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ae680 x1874531139522176/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695785.657860:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695785.657879:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139522176:192.168.201.140@tcp:38:kworker.0 00000100:00100000:2.0:1787695785.657880:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465af480 x1874531139522432/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695785.657899:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695785.657903:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695785.657904:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787695785.657908:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695785.657913:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ae680 x1874531139522176/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695785.657918:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000071cfa937] to pc [ptlrpcd_rcv+-1] req@ffff8b14465af480 x1874531139522432/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.657953:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695785.657957:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1787695785.657962:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695785.657963:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14475f9c00. 00000100:00080000:2.0:1787695785.657973:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1787695785.657974:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695785.657980:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000100:00080000:2.0:1787695785.657987:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000020:00000040:0.0:1787695785.657988:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695785.657995:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:2.0:1787695785.657995:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695785.657998:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695785.658001:0:47074:0:(import.c:485:import_select_connection()) Process entered 02000000:00000010:0.0:1787695785.658006:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445662c00. 00000100:00080000:2.0:1787695785.658008:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3576 02000000:00000001:0.0:1787695785.658013:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695785.658015:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000400:00000200:2.0:1787695785.658018:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000010:0.0:1787695785.658020:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ae680. 00000400:00000200:2.0:1787695785.658024:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:0.0:1787695785.658025:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695785.658033:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695785.658035:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695785.658037:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695785.658042:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695785.658045:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695785.658047:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695785.658048:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695785.658052:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787695785.658053:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695785.658062:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000100:00000001:0.0:1787695785.658063:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787695785.658068:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787695785.658069:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695785.658080:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:2.0:1787695785.658080:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:2.0:1787695785.658087:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:0.0:1787695785.658091:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ac700 x1874531139522304/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787695785.658093:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:2.0:1787695785.658106:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695785.658115:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695785.658116:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695785.658131:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695785.658135:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:2.0:1787695785.658137:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695785.658138:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695785.658141:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695785.658149:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139522304:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695785.658152:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695785.658154:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:2.0:1787695785.658163:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 02000000:00000001:0.0:1787695785.658164:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695785.658169:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695785.658174:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1787695785.658174:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695785.658183:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695785.658184:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14475f9c00. 02000000:00000001:0.0:1787695785.658192:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695785.658192:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695785.658199:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1787695785.658208:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:2.0:1787695785.658209:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695785.658216:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:0.0:1787695785.658218:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b9a0. 00000008:00100000:2.0:1787695785.658221:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000010:0.0:1787695785.658225:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96330. 00000008:00000001:2.0:1787695785.658226:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695785.658231:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695785.658232:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139522304, portal 10 00000100:00000001:0.0:1787695785.658239:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695785.658242:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928802048 : -128555780749568 : ffff8b14465ac700) 00000100:00000010:2.0:1787695785.658242:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ae680. 00000020:00000040:2.0:1787695785.658254:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=26 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000040:0.0:1787695785.658261:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ac700 x1874531139522304/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695785.658261:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695785.658266:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695785.658276:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695785.658281:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695785.658284:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695785.658287:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661800. 00000100:00000001:2.0:1787695785.658297:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.658306:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1787695785.658313:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ae680 x1874531139522560/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1787695785.658321:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695785.658327:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000010:0.0:1787695785.658329:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96550. 00000100:00000001:2.0:1787695785.658330:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695785.658336:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019831621] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ae680 x1874531139522560/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695785.658339:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139522304, offset 0 00000100:00000001:2.0:1787695785.658355:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695785.658361:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:2.0:1787695785.658362:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1787695785.658398:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:0.0:1787695785.658399:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:2.0:1787695785.658407:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1787695785.658411:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000040:2.0:1787695785.658424:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:0.0:1787695785.658425:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695785.658435:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000040:2.0:1787695785.658437:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3683) 00000400:00000200:0.0:1787695785.658438:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695785.658449:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695785.658454:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16890:16890:256:4294967295] 192.168.201.140@tcp LPNI seq info [16889:16889:8:4294967295] 00000400:00000200:0.0:1787695785.658457:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695785.658462:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695785.658465:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695785.658468:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:0.0:1787695785.658471:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695785.658473:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 1, 1 secs later) 00000800:00000200:0.1:1787695785.658477:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695785.658479:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 1, 1 secs later) 00000800:00000100:0.0:1787695785.658482:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695785.658483:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:0.0:1787695785.658486:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695785.658488:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695785.658490:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96550 00000400:00000010:0.0:1787695785.658492:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96550. 00000100:00000001:0.0:1787695785.658493:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695785.658496:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ac700 x1874531139522304/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.658502:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695785.658504:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ac700 x1874531139522304/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695795 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.658518:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.658519:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695785.658522:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695785.658524:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:0.0:1787695785.658527:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695785.658528:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298346121 00000100:00000001:0.0:1787695785.658530:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298351929 00000100:00000001:0.0:1787695785.658532:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298346121 00000100:00000001:0.0:1787695785.658533:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298351929 00000100:00000001:0.0:1787695785.658534:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695785.658536:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.658537:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695785.658539:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695785/real 1787695785] req@ffff8b14465ac700 x1874531139522304/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695785.658545:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695785.658546:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695785.658547:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b9a0. 00000400:00000200:0.0:1787695785.658548:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96330 00000400:00000010:0.0:1787695785.658549:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96330. 00000100:00000001:0.0:1787695785.658551:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695785.658553:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ac700 x1874531139522304/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695785.658558:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ac700 x1874531139522304/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.658562:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695785.658563:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.658564:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695785.658565:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695785.658567:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ac700 x1874531139522304/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.658575:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695785.658577:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ac700 x1874531139522304/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695795 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.658582:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695785.658583:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.658584:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695785.658585:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.658586:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695785.658587:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695785.658589:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695785.658591:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695785.658592:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695785.658595:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ac700 x1874531139522304/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695795 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695785.658600:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139522304:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695785.658602:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695785.658605:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465af480 x1874531139522432/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.658609:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695785.658610:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695785.658611:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695785.658612:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695785.658614:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695785.658615:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139522432:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695785.658617:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695785.658619:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695785.658623:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695785.658624:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695785.658627:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14475f8800. 02000000:00000001:0.0:1787695785.658628:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695785.658630:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695785.658632:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b9a0. 00000400:00000010:0.0:1787695785.658634:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96330. 00000100:00000200:0.0:1787695785.658636:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139522432, portal 10 00000100:00000001:0.0:1787695785.658637:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695785.658638:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928813696 : -128555780737920 : ffff8b14465af480) 00000100:00000040:0.0:1787695785.658643:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465af480 x1874531139522432/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.658647:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695785.658649:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695785.658650:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96550. 00000100:00000200:0.0:1787695785.658652:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139522432, offset 0 00000400:00000200:0.0:1787695785.658654:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695785.658657:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695785.658659:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695785.658661:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695785.658662:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695785.658664:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695785.658665:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695785.658668:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16891:16891:256:4294967295] 192.168.201.140@tcp LPNI seq info [16890:16890:8:4294967295] 00000400:00000200:0.0:1787695785.658671:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695785.658675:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695785.658678:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695785.658680:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:0.0:1787695785.658682:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695785.658684:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 1, 1 secs later) 00000800:00000200:0.1:1787695785.658687:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695785.658689:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 1, 1 secs later) 00000800:00000100:0.0:1787695785.658691:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695785.658697:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:0.0:1787695785.658699:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695785.658701:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695785.658704:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96550 00000400:00000010:0.0:1787695785.658705:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96550. 00000100:00000001:0.0:1787695785.658706:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695785.658708:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465af480 x1874531139522432/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695810 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.658714:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695785.658716:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465af480 x1874531139522432/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695810 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.658720:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.658721:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695785.658723:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695785.658725:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:0.0:1787695785.658727:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695785.658729:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298346009 00000100:00000001:0.0:1787695785.658730:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298351929 00000100:00000001:0.0:1787695785.658732:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298346009 00000100:00000001:0.0:1787695785.658733:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298351929 00000100:00000001:0.0:1787695785.658735:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695785.658736:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.658737:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695785.658740:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695785/real 1787695785] req@ffff8b14465af480 x1874531139522432/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695785.658745:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695785.658746:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695785.658747:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b9a0. 00000400:00000200:0.0:1787695785.658750:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96330 00000400:00000010:0.0:1787695785.658752:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96330. 00000100:00000001:0.0:1787695785.658759:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695785.658763:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465af480 x1874531139522432/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695785.658771:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465af480 x1874531139522432/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.658778:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695785.658780:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.658782:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695785.658784:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695785.658788:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465af480 x1874531139522432/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.658796:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695785.658800:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465af480 x1874531139522432/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.658807:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695785.658810:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.658812:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695785.658813:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.658815:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695785.658816:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695785.658819:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695785.658823:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695785.658825:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695785.658830:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465af480 x1874531139522432/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695785.658839:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139522432:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695785.658843:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.658849:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695785.658852:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ac700 x1874531139522304/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695795 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.658856:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695785.658858:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695785.658859:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14475f9c00. 02000000:00000001:0.0:1787695785.658861:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695785.658862:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695785.658863:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=24 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695785.658865:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695785.658866:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663c00. 02000000:00000001:0.0:1787695785.658868:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695785.658869:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695785.658870:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ac700. 00000100:00000001:0.0:1787695785.658872:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695785.658873:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.658874:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695785.658876:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465af480 x1874531139522432/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695810 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.658880:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695785.658881:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695785.658882:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14475f8800. 02000000:00000001:0.0:1787695785.658884:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695785.658885:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695785.658885:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695785.658886:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695785.658887:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445662400. 02000000:00000001:0.0:1787695785.658888:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695785.658889:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695785.658890:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465af480. 00000100:00000001:0.0:1787695785.658892:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695785.658893:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.658894:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.658897:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695785.658897:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.658899:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695785.658903:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695785.658904:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695785.658907:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ae680 x1874531139522560/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.658912:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695785.658913:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695785.658914:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695785.658916:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695785.658917:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695785.658919:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139522560:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695785.658921:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695785.658923:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695785.658924:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695785.658925:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695785.658927:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14475f8800. 02000000:00000001:0.0:1787695785.658929:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695785.658931:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695785.658933:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b9a0. 00000400:00000010:0.0:1787695785.658935:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96330. 00000100:00000200:0.0:1787695785.658937:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139522560, portal 10 00000100:00000001:0.0:1787695785.658939:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695785.658940:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928810112 : -128555780741504 : ffff8b14465ae680) 00000100:00000040:0.0:1787695785.658944:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ae680 x1874531139522560/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.658948:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695785.658950:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695785.658951:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96550. 00000100:00000200:0.0:1787695785.658953:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139522560, offset 0 00000400:00000200:0.0:1787695785.658955:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695785.658958:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695785.658960:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695785.658962:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695785.658964:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695785.658969:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695785.658971:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695785.658973:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16892:16892:256:4294967295] 192.168.201.140@tcp LPNI seq info [16891:16891:8:4294967295] 00000400:00000200:0.0:1787695785.658976:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695785.658980:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695785.658984:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695785.658986:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:0.0:1787695785.658988:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695785.658990:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 1, 1 secs later) 00000800:00000200:0.1:1787695785.658993:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695785.658995:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 1, 1 secs later) 00000800:00000100:0.0:1787695785.658997:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695785.658999:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:0.0:1787695785.659001:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695785.659003:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695785.659005:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96550 00000400:00000010:0.0:1787695785.659006:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96550. 00000100:00000001:0.0:1787695785.659008:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695785.659010:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ae680 x1874531139522560/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695795 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.659015:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695785.659017:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ae680 x1874531139522560/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695795 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.659021:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.659022:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695785.659025:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695785.659026:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:0.0:1787695785.659028:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695785.659030:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298345987 00000100:00000001:0.0:1787695785.659032:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298351929 00000100:00000001:0.0:1787695785.659033:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298345987 00000100:00000001:0.0:1787695785.659039:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298351929 00000100:00000001:0.0:1787695785.659040:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695785.659042:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.659043:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695785.659046:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695785/real 1787695785] req@ffff8b14465ae680 x1874531139522560/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695785.659051:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695785.659052:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695785.659053:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b9a0. 00000400:00000200:0.0:1787695785.659055:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96330 00000400:00000010:0.0:1787695785.659056:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96330. 00000100:00000001:0.0:1787695785.659057:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695785.659059:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ae680 x1874531139522560/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695785.659065:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ae680 x1874531139522560/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.659068:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695785.659069:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.659070:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695785.659071:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695785.659073:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ae680 x1874531139522560/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.659077:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695785.659080:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ae680 x1874531139522560/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695795 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.659084:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695785.659085:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.659086:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695785.659087:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.659091:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695785.659092:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695785.659094:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695785.659096:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695785.659098:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695785.659101:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ae680 x1874531139522560/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695795 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695785.659105:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139522560:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695785.659107:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.659109:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695785.659110:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ae680 x1874531139522560/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695795 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695785.659114:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695785.659115:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695785.659117:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14475f8800. 02000000:00000001:0.0:1787695785.659132:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695785.659133:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695785.659134:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=25 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695785.659136:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695785.659137:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661800. 02000000:00000001:0.0:1787695785.659139:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695785.659140:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695785.659141:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ae680. 00000100:00000001:0.0:1787695785.659143:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695785.659144:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.659145:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.659147:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695785.659148:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695785.659150:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695785.659151:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695785.659152:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695785.659153:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695785.910438:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695785.910444:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695785.910446:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695785.910450:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695785.910452:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695785.910470:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695785.910472:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695785.910474:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695785.910476:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695786.358448:0:2403:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695786.358467:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695786.358469:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=39 : 39 : 27) 00000100:00000001:0.0:1787695786.358476:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695786.358479:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695786.358486:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695786.358489:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695786.550438:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695786.550444:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695786.550449:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695786.550450:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695786.550453:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695786.550456:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695786.550458:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695786.550463:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695786.550465:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695786.550466:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695786.550468:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695786.550470:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695786.550472:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695786.550473:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695786.550476:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695786.550477:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695786.550477:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695786.550479:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695786.550479:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695786.550480:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695786.550481:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695786.550482:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695786.550483:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695786.550484:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695786.550486:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695786.550487:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695786.550489:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695786.550490:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695786.550491:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695786.550492:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695786.550493:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695786.550494:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695786.550494:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695786.550495:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695786.550496:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695786.550497:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695786.550537:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695786.550540:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695786.550543:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787695786.678446:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695786.678457:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695786.678460:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695786.678464:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695786.678466:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695786.678468:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695786.678470:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695786.934467:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695786.934481:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695786.934483:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695786.934487:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695786.934490:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695786.934493:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695786.934495:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695786.934496:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695786.934498:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695787.574456:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695787.574461:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695787.574465:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695787.574481:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695787.574484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695787.574485:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695787.574487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695787.574488:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695787.574490:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695787.574491:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695787.574493:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695787.574494:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695787.574583:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695787.574588:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695787.574591:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787695787.702427:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695787.702438:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695787.702440:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695787.702444:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695787.702446:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695787.702449:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695787.702451:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695787.958420:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695787.958426:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695787.958427:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695787.958432:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695787.958434:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695787.958437:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695787.958439:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695787.958441:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695787.958443:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695788.598481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695788.598486:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695788.598491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695788.598492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695788.598495:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695788.598496:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695788.598498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695788.598499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695788.598501:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695788.598502:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695788.598504:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695788.598505:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695788.598586:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695788.598594:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695788.598600:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787695788.726461:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695788.726474:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695788.726477:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695788.726484:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695788.726486:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695788.726491:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695788.726494:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695788.982435:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695788.982441:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695788.982443:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695788.982446:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695788.982449:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695788.982452:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695788.982453:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695788.982456:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695788.982458:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695789.622463:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695789.622469:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695789.622473:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695789.622474:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695789.622477:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695789.622478:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695789.622481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695789.622481:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695789.622484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695789.622484:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695789.622487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695789.622487:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695789.622574:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695789.622580:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695789.622584:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787695789.750464:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695789.750476:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695789.750478:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695789.750482:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695789.750484:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695789.750488:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695789.750507:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695790.006420:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695790.006443:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695790.006444:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695790.006448:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695790.006451:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695790.006454:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695790.006456:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695790.006460:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695790.006463:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695790.646487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695790.646492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695790.646497:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695790.646501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695790.646505:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695790.646522:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695790.646525:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695790.646527:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695790.646529:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695790.646530:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695790.646532:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695790.646533:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695790.646593:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695790.646596:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695790.646601:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787695790.774506:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695790.774519:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695790.774521:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695790.774526:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695790.774527:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695790.774530:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695790.774532:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695790.966431:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695790.966440:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1787695790.966442:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695790.966444:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787695790.966447:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695790.966449:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695790.966453:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695790.966459:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3678 00000400:00000200:2.0:1787695790.966466:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695790.966470:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695790.966473:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695790.966475:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695790.966478:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695790.966480:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695790.966481:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695790.966484:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695790.966485:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:2.0:1787695790.966488:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695790.966490:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:2.0:1787695790.966492:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:2.0:1787695790.966494:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:2.0:1787695790.966496:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695790.966498:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695790.966499:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695790.966500:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695790.966502:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:2.0:1787695790.966513:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695790.966515:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695790.966518:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695790.966523:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695790.966525:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695790.966527:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695790.966530:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787695790.966533:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695790.966534:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695790.966540:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ad880. 00000020:00000040:2.0:1787695790.966542:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:2.0:1787695790.966545:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695790.966547:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695790.966549:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695790.966551:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695790.966555:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695790.966561:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445660000. 00000100:00000001:2.0:1787695790.966567:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695790.966574:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ad880 x1874531139522688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695790.966583:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695790.966586:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695790.966589:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000075b4d632] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ad880 x1874531139522688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695790.966605:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695790.966608:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695790.966614:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787695790.966615:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695790.966616:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695790.966618:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695790.966620:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3678 00000400:00000200:2.0:1787695790.966628:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695790.966631:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00001000:2.0:1787695790.966634:0:47074:0:(import.c:2149:obd_at_measure()) add 10 to ffff8b1458081c18 time=106 v=5 (1 0 0 0) 00000100:00001000:2.0:1787695790.966637:0:47074:0:(import.c:2215:obd_at_measure()) AT ffff8b1458081c18 change: old=5 new=10 delta=5 (val=10) hist 10 0 0 0 00000100:00000001:0.0:1787695790.966640:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1787695790.966640:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 5s 00000100:00000040:2.0:1787695790.966645:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695790.966646:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695790.966647:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695790.966650:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695790.966651:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695790.966654:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695790.966656:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695790.966658:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000100:00000040:0.0:1787695790.966660:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ad880 x1874531139522688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787695790.966660:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695790.966662:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:2.0:1787695790.966667:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:2.0:1787695790.966672:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000001:0.0:1787695790.966680:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:2.0:1787695790.966680:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695790.966684:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695790.966685:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695790.966688:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695790.966690:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787695790.966691:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:0.0:1787695790.966694:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1787695790.966695:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 02000000:00000001:0.0:1787695790.966698:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695790.966701:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00100000:0.0:1787695790.966706:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139522688:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695790.966706:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695790.966710:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787695790.966713:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695790.966715:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695790.966718:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695790.966719:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695790.966722:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787695790.966723:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787695790.966724:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787695790.966726:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:2.0:1787695790.966728:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695790.966731:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695790.966734:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ace00. 00000020:00000040:2.0:1787695790.966739:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=25 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000010:0.0:1787695790.966740:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14475f9800. 02000000:00000001:2.0:1787695790.966741:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695790.966745:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695790.966746:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695790.966747:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695790.966748:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695790.966753:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695790.966756:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000010:2.0:1787695790.966757:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661800. 00000100:00000001:2.0:1787695790.966762:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695790.966763:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b9a0. 00000100:00100000:2.0:1787695790.966769:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ace00 x1874531139522816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:0.0:1787695790.966775:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96330. 00000100:00000001:2.0:1787695790.966776:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695790.966781:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695790.966784:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046eddc29] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ace00 x1874531139522816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695790.966786:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139522688, portal 10 00000100:00000001:0.0:1787695790.966794:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695790.966796:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695790.966798:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928806528 : -128555780745088 : ffff8b14465ad880) 00000100:00080000:2.0:1787695790.966799:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695790.966805:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787695790.966807:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695790.966808:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695790.966810:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:0.0:1787695790.966814:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ad880 x1874531139522688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695790.966815:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3678 00000400:00000200:2.0:1787695790.966818:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695790.966820:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695790.966822:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695790.966826:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695790.966831:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695790.966834:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695790.966844:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787695790.966846:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695790.966851:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000040:0.0:1787695790.966852:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000020:00000040:2.0:1787695790.966853:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000400:00000010:0.0:1787695790.966857:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96550. 00000020:00000001:2.0:1787695790.966862:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000200:0.0:1787695790.966864:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139522688, offset 0 00000020:00000040:2.0:1787695790.966864:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:2.0:1787695790.966871:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:2.0:1787695790.966873:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:2.0:1787695790.966878:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695790.966880:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695790.966880:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695790.966885:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695790.966887:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695790.966889:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:2.0:1787695790.966891:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:0.0:1787695790.966894:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695790.966895:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695790.966898:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695790.966901:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1787695790.966903:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 02000000:00000001:2.0:1787695790.966905:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695790.966910:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695790.966912:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000400:00000200:0.0:1787695790.966913:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000008:00000001:2.0:1787695790.966916:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695790.966918:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695790.966920:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000010:2.0:1787695790.966923:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465af800. 00000400:00000200:0.0:1787695790.966925:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000020:00000040:2.0:1787695790.966929:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000400:00000200:0.0:1787695790.966930:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 02000000:00000001:2.0:1787695790.966934:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695790.966937:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695790.966938:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16893:16893:256:4294967295] 192.168.201.140@tcp LPNI seq info [16892:16892:8:4294967295] 00000100:00000001:2.0:1787695790.966941:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695790.966944:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695790.966946:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695790.966947:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000010:2.0:1787695790.966951:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445662400. 00000100:00000001:2.0:1787695790.966955:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695790.966962:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00100000:2.0:1787695790.966964:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465af800 x1874531139522944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.0:1787695790.966973:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695790.966979:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000010:0.0:1787695790.966982:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000100:00000001:2.0:1787695790.966982:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695790.966990:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b410802f] to pc [ptlrpcd_rcv+-1] req@ffff8b14465af800 x1874531139522944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1787695790.966994:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695790.967004:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:2.0:1787695790.967012:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695790.967014:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695790.967016:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1787695790.967017:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695790.967018:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695790.967020:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695790.967022:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3678 00000100:00000001:0.0:1787695790.967025:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695790.967038:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:0.0:1787695790.967047:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000400:00000200:2.0:1787695790.967047:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695790.967053:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1787695790.967057:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:2.0:1787695790.967059:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695790.967061:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298352581 00000100:00000001:2.0:1787695790.967062:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695790.967067:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695790.967070:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695790.967071:0:2402:0:(niobuf.c:685:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, simply extend its deadline from 4298351929 00000020:00000001:2.0:1787695790.967071:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695790.967074:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000100:00000001:0.0:1787695790.967078:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298352581 00000020:00000001:2.0:1787695790.967078:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695790.967082:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000100:00000001:0.0:1787695790.967085:0:2402:0:(niobuf.c:685:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, simply extend its deadline from 4298351929 00000020:00000001:2.0:1787695790.967085:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000001:0.0:1787695790.967091:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1787695790.967091:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:2.0:1787695790.967096:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695790.967097:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695790.967101:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695790.967103:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695790.967105:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695790.967107:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695790.967108:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695790.967112:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000400:00000200:1.0:1787695790.967114:0:2391:0:(lib-socket.c:438:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.140:988 00000100:00000001:0.0:1787695790.967115:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1787695790.967116:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695790.967121:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695790.967124:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695790.967127:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695790.967131:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787695790.967131:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787695790.967133:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787695790.967136:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1787695790.967141:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ace00 x1874531139522816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695790.967141:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695790.967146:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695790.967150:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787695790.967157:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:2.0:1787695790.967157:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695790.967161:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695790.967162:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695790.967168:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:2.0:1787695790.967170:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465adf80. 02000000:00000001:0.0:1787695790.967171:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1787695790.967174:0:2391:0:(lib-socket.c:438:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.140:988 02000000:00000001:0.0:1787695790.967176:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1787695790.967177:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 00000100:00100000:0.0:1787695790.967184:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139522816:192.168.201.140@tcp:38:kworker.0 02000000:00000001:2.0:1787695790.967185:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695790.967189:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695790.967190:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695790.967193:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695790.967195:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695790.967195:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695790.967199:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787695790.967200:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:0.0:1787695790.967203:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14475fbc00. 02000000:00000001:2.0:1787695790.967205:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695790.967206:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695790.967212:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000010:2.0:1787695790.967214:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663c00. 00000400:00000010:0.0:1787695790.967215:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b6e0. 00000400:00000010:0.0:1787695790.967223:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96b28. 00000100:00000001:2.0:1787695790.967223:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695790.967230:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139522816, portal 10 00000100:00000001:0.0:1787695790.967233:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:2.0:1787695790.967235:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465adf80 x1874531139523072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695790.967236:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928803840 : -128555780747776 : ffff8b14465ace00) 00000100:00000040:0.0:1787695790.967247:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ace00 x1874531139522816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695805 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695790.967247:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695790.967250:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695790.967260:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695790.967260:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ee5f8f2] to pc [ptlrpcd_rcv+-1] req@ffff8b14465adf80 x1874531139523072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695790.967268:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695790.967272:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96198. 00000100:00000200:0.0:1787695790.967276:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139522816, offset 0 00000400:00000200:0.0:1787695790.967283:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695790.967283:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695790.967289:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1787695790.967299:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:2.0:1787695790.967302:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000400:00000200:0.0:1787695790.967305:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:2.0:1787695790.967307:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695790.967309:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1787695790.967315:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1787695790.967317:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:0.0:1787695790.967319:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00080000:2.0:1787695790.967323:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3678 00000400:00000200:0.0:1787695790.967325:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695790.967330:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695790.967333:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695790.967337:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16894:16894:255:4294967295] 192.168.201.140@tcp LPNI seq info [16893:16893:7:4294967295] 00000400:00000200:2.0:1787695790.967341:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695790.967345:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1787695790.967346:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695790.967355:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000040:2.0:1787695790.967359:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695790.967382:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695790.967386:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695790.967388:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695790.967391:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695790.967392:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:2.0:1787695790.967394:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695790.967396:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000020:00000001:2.0:1787695790.967398:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000020:00000001:2.0:1787695790.967400:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000040:2.0:1787695790.967402:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695790.967404:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695790.967405:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695790.967406:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000200:0.0:1787695790.967409:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000020:00000040:2.0:1787695790.967411:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:2.0:1787695790.967416:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000800:00000010:0.0:1787695790.967417:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da800. 00000100:00000001:2.0:1787695790.967422:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695790.967425:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:0.0:1787695790.967426:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:2.0:1787695790.967429:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:0.1:1787695790.967434:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:2.0:1787695790.967435:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695790.967439:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695790.967440:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695790.967443:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000040:0.0:1787695790.967445:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000008:00000001:2.0:1787695790.967446:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695790.967450:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695790.967452:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695790.967456:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298351430 00000100:00000010:2.0:1787695790.967456:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ac000. 00000020:00000040:2.0:1787695790.967461:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000100:00000001:0.0:1787695790.967462:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298352581 00000100:00000001:0.0:1787695790.967467:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298351430 02000000:00000001:2.0:1787695790.967468:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787695790.967472:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298352581 02000000:00000001:2.0:1787695790.967472:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695790.967475:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787695790.967478:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695790.967478:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787695790.967482:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695790.967482:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695790.967487:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000010:2.0:1787695790.967488:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445662c00. 00000100:00000040:0.0:1787695790.967491:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465af800 x1874531139522944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695790.967498:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695790.967501:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695790.967503:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695790.967506:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ac000 x1874531139523200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787695790.967507:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695790.967510:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695790.967515:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695790.967523:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139522944:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787695790.967524:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695790.967528:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695790.967529:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:2.0:1787695790.967534:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a231b497] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ac000 x1874531139523200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787695790.967539:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695790.967543:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695790.967544:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695790.967546:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1787695790.967548:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000010:0.0:1787695790.967551:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14475f9c00. 00000100:00080000:2.0:1787695790.967552:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787695790.967555:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:0.0:1787695790.967556:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695790.967558:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695790.967562:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:0.0:1787695790.967564:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:2.0:1787695790.967567:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3678 00000400:00000010:0.0:1787695790.967569:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b7e8. 00000400:00000200:2.0:1787695790.967573:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695790.967574:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000400:00000200:2.0:1787695790.967578:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000200:0.0:1787695790.967580:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139522944, portal 4 00000100:00001000:2.0:1787695790.967581:0:47074:0:(import.c:2149:obd_at_measure()) add 10 to ffff8b144515c418 time=106 v=5 (1 0 0 0) 00000100:00000001:0.0:1787695790.967583:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00001000:2.0:1787695790.967586:0:47074:0:(import.c:2215:obd_at_measure()) AT ffff8b144515c418 change: old=5 new=10 delta=5 (val=10) hist 10 0 0 0 00000100:00000001:0.0:1787695790.967587:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928814592 : -128555780737024 : ffff8b14465af800) 00000100:00080000:2.0:1787695790.967593:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 5s 00000100:00000040:2.0:1787695790.967597:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:0.0:1787695790.967598:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465af800 x1874531139522944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695790.967601:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695790.967605:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695790.967611:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695790.967612:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1787695790.967614:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695790.967617:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000100:00000040:0.0:1787695790.967620:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000020:00000001:2.0:1787695790.967620:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695790.967623:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000400:00000010:0.0:1787695790.967624:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96908. 00000020:00000001:2.0:1787695790.967627:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000200:0.0:1787695790.967628:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139522944, offset 0 00000020:00000001:2.0:1787695790.967631:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000400:00000200:0.0:1787695790.967633:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695790.967640:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:2.0:1787695790.967644:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695790.967646:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:2.0:1787695790.967648:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1787695790.967653:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1787695790.967654:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695790.967658:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:2.0:1787695790.967659:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:0.0:1787695790.967664:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000020:00000040:2.0:1787695790.967664:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000400:00000200:0.0:1787695790.967669:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00080000:2.0:1787695790.967671:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:0.0:1787695790.967677:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16895:16895:254:4294967295] 192.168.201.140@tcp LPNI seq info [16894:16894:6:4294967295] 00000100:00000001:2.0:1787695790.967677:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695790.967683:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1787695790.967686:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:2.0:1787695790.967687:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695790.967691:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695790.967696:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:2.0:1787695790.967696:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695790.967698:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000800:00000200:0.0:1787695790.967703:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000008:00000001:2.0:1787695790.967703:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695790.967707:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1787695790.967710:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da700. 00000100:00000010:2.0:1787695790.967712:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465aed80. 00000800:00000200:0.0:1787695790.967717:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000020:00000040:2.0:1787695790.967718:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=26 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:2.0:1787695790.967724:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695790.967726:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1787695790.967727:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:2.0:1787695790.967728:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787695790.967732:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695790.967733:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695790.967736:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695790.967737:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 02000000:00000010:2.0:1787695790.967740:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663800. 00000100:00000001:2.0:1787695790.967743:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695790.967750:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00100000:2.0:1787695790.967750:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465aed80 x1874531139523328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695790.967755:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298351460 00000100:00000001:0.0:1787695790.967761:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298352581 00000100:00000001:2.0:1787695790.967765:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695790.967767:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298351460 00000100:00000001:2.0:1787695790.967768:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695790.967774:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298352581 00000100:00000040:2.0:1787695790.967775:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bed91c58] to pc [ptlrpcd_rcv+-1] req@ffff8b14465aed80 x1874531139523328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695790.967781:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695790.967789:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695790.967790:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695790.967792:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695790.967796:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695790.967801:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695790.967802:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1787695790.967807:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695790.967808:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:2.0:1787695790.967810:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1787695790.967813:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695790.967816:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3688) 00000100:00000001:0.0:1787695790.967818:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695790.967823:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695790.967825:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695790.967831:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465adf80 x1874531139523072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695790.967840:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695790.967841:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695790.967844:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695790.967845:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695790.967848:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695790.967850:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139523072:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695790.967854:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695790.967856:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695790.967857:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695790.967859:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695790.967862:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14475f8800. 02000000:00000001:0.0:1787695790.967864:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695790.967868:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695790.967870:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b8f0. 00000400:00000010:0.0:1787695790.967873:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f965d8. 00000100:00000200:0.0:1787695790.967876:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139523072, portal 4 00000100:00000001:0.0:1787695790.967884:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695790.967886:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928808320 : -128555780743296 : ffff8b14465adf80) 00000100:00000040:0.0:1787695790.967892:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465adf80 x1874531139523072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695790.967899:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695790.967901:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695790.967904:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96f68. 00000100:00000200:0.0:1787695790.967906:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139523072, offset 0 00000400:00000200:0.0:1787695790.967910:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695790.967915:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695790.967918:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695790.967921:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695790.967924:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695790.967926:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695790.967928:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695790.967931:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16896:16896:253:4294967295] 192.168.201.140@tcp LPNI seq info [16895:16895:5:4294967295] 00000400:00000200:0.0:1787695790.967935:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695790.967942:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695790.967946:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695790.967949:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460daa00. 00000800:00000200:0.0:1787695790.967953:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695790.967957:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695790.967959:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695790.967962:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:0.0:1787695790.967965:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695790.967967:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298351132 00000100:00000001:0.0:1787695790.967970:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298352581 00000100:00000001:0.0:1787695790.967972:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298351132 00000100:00000001:0.0:1787695790.967974:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298352581 00000100:00000001:0.0:1787695790.967976:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695790.967982:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695790.967984:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695790.967989:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ac000 x1874531139523200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695790.967996:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695790.967997:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695790.967999:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695790.968001:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695790.968003:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695790.968006:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139523200:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695790.968009:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695790.968011:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695790.968013:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695790.968014:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695790.968018:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445435c00. 02000000:00000001:0.0:1787695790.968020:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695790.968023:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695790.968025:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617ba50. 00000400:00000010:0.0:1787695790.968043:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96110. 00000100:00000200:0.0:1787695790.968046:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139523200, portal 10 00000100:00000001:0.0:1787695790.968048:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695790.968049:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928800256 : -128555780751360 : ffff8b14465ac000) 00000100:00000040:0.0:1787695790.968055:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ac000 x1874531139523200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695790.968062:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695790.968064:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695790.968066:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96cc0. 00000100:00000200:0.0:1787695790.968068:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139523200, offset 0 00000400:00000200:0.0:1787695790.968073:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695790.968076:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695790.968080:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695790.968083:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695790.968090:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695790.968093:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695790.968095:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695790.968098:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16897:16897:252:4294967295] 192.168.201.140@tcp LPNI seq info [16896:16896:4:4294967295] 00000400:00000200:0.0:1787695790.968102:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695790.968108:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695790.968112:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695790.968115:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da500. 00000800:00000200:0.0:1787695790.968119:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695790.968123:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695790.968125:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695790.968128:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:0.0:1787695790.968131:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695790.968133:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298351318 00000100:00000001:0.0:1787695790.968136:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298352581 00000100:00000001:0.0:1787695790.968138:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298351318 00000100:00000001:0.0:1787695790.968140:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298352581 00000100:00000001:0.0:1787695790.968142:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695790.968144:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695790.968146:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695790.968149:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465aed80 x1874531139523328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695790.968156:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695790.968158:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695790.968160:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695790.968162:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695790.968164:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695790.968166:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139523328:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695790.968169:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695790.968171:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695790.968177:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695790.968179:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695790.968184:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14514b6c00. 02000000:00000001:0.0:1787695790.968186:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695790.968190:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695790.968192:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b528. 00000400:00000010:0.0:1787695790.968196:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96770. 00000100:00000200:0.0:1787695790.968198:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139523328, portal 10 00000100:00000001:0.0:1787695790.968200:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695790.968202:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928811904 : -128555780739712 : ffff8b14465aed80) 00000100:00000040:0.0:1787695790.968207:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465aed80 x1874531139523328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695805 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695790.968213:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695790.968216:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695790.968218:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96dd0. 00000100:00000200:0.0:1787695790.968220:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139523328, offset 0 00000400:00000200:0.0:1787695790.968224:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695790.968228:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695790.968231:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695790.968233:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695790.968236:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695790.968239:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695790.968241:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695790.968244:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16898:16898:251:4294967295] 192.168.201.140@tcp LPNI seq info [16897:16897:3:4294967295] 00000400:00000200:0.0:1787695790.968248:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695790.968254:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695790.968258:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695790.968261:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1445678300. 00000800:00000200:0.0:1787695790.968263:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695790.968266:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695790.968268:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695790.968270:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:0.0:1787695790.968272:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695790.968278:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298351296 00000100:00000001:0.0:1787695790.968280:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298352581 00000100:00000001:0.0:1787695790.968282:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298351296 00000100:00000001:0.0:1787695790.968283:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298352581 00000100:00000001:0.0:1787695790.968285:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695790.968286:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695790.968288:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695790.968289:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695790.968292:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695790.968293:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695790.968294:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695790.968295:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695790.968296:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695790.968298:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695791.030459:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695791.030465:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695791.030467:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695791.030471:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695791.030473:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695791.030476:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695791.030478:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695791.030480:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695791.030482:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695791.670464:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695791.670470:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695791.670480:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695791.670481:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695791.670484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695791.670484:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695791.670488:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695791.670489:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695791.670491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695791.670494:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695791.670496:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695791.670501:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695791.670502:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695791.670503:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695791.670506:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695791.670507:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695791.670509:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695791.670511:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695791.670513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695791.670514:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695791.670564:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695791.670569:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695791.670572:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787695791.798621:0:2391:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.201.140:988 00000400:00000100:1.0:1787695791.798664:0:2391:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.140@tcp at host 192.168.201.140 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1787695791.798672:0:2391:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695791.798675:0:2391:0:(router.c:1791:lnet_notify()) 192.168.201.40@tcp notifying 192.168.201.140@tcp: down 00000400:00000200:1.0:1787695791.798680:0:2391:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695791.798683:0:2391:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:1.0:1787695791.798686:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:1.0:1787695791.798703:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695791.798708:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1787695791.798711:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1787695791.798717:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96550 00000400:00000010:1.0:1787695791.798719:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96550. 00000100:00000001:1.0:1787695791.798724:0:2391:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695791.798733:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14465ad880 x1874531139522688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695791.798758:0:2391:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695791.798763:0:2391:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ad880 x1874531139522688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695815 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695791.798771:0:2391:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695791.798773:0:2391:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787695791.798784:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:1.0:1787695791.798787:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da800. 00000400:00000200:1.0:1787695791.798790:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1787695791.798795:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1787695791.798799:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96198 00000400:00000010:1.0:1787695791.798802:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96198. 00000100:00000001:1.0:1787695791.798805:0:2391:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695791.798810:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14465ace00 x1874531139522816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695805 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.798815:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695791.798822:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695791.798824:0:2391:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787695791.798830:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:1.0:1787695791.798831:0:2391:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ace00 x1874531139522816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695805 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695791.798843:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695790/real 1787695791] req@ffff8b14465ace00 x1874531139522816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695805 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695791.798844:0:2391:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695791.798848:0:2391:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787695791.798860:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:1.0:1787695791.798865:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da700. 00000400:00000001:0.0:1787695791.798866:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:1.0:1787695791.798869:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000001:0.0:1787695791.798870:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695791.798876:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b6e0. 00000400:00000200:1.0:1787695791.798876:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1787695791.798883:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96b28 00000400:00000010:0.0:1787695791.798887:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96b28. 00000100:00000001:0.0:1787695791.798892:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:1.0:1787695791.798892:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96908 00000400:00000010:1.0:1787695791.798898:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96908. 00000100:00000200:0.0:1787695791.798901:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ace00 x1874531139522816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695791.798904:0:2391:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695791.798915:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14465af800 x1874531139522944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695791.798920:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ace00 x1874531139522816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695791.798938:0:2391:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695791.798945:0:2391:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465af800 x1874531139522944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695815 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.798964:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695791.798967:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695791.798971:0:2391:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695791.798973:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695791.798976:0:2391:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695791.798977:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695791.798988:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ace00 x1874531139522816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1787695791.798988:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:1.0:1787695791.798995:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460daa00. 00000400:00000200:1.0:1787695791.799001:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:0.0:1787695791.799004:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1787695791.799010:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000100:00000040:0.0:1787695791.799029:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ace00 x1874531139522816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695805 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695791.799030:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96f68 00000400:00000010:1.0:1787695791.799034:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96f68. 00000100:00000001:1.0:1787695791.799040:0:2391:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787695791.799046:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000200:1.0:1787695791.799049:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14465adf80 x1874531139523072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.799051:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.799057:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695791.799059:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.799065:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695791.799069:0:2391:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787695791.799070:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:1.0:1787695791.799076:0:2391:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465adf80 x1874531139523072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695815 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787695791.799077:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695791.799089:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695791.799090:0:2391:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695791.799095:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1787695791.799095:0:2391:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1787695791.799106:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ace00 x1874531139522816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695805 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1787695791.799107:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:1.0:1787695791.799115:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da500. 00000400:00000200:1.0:1787695791.799121:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00100000:0.0:1787695791.799128:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139522816:192.168.201.140@tcp:38:kworker.0 00000400:00000200:1.0:1787695791.799131:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1787695791.799140:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96cc0 00000100:00000001:0.0:1787695791.799141:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000400:00000010:1.0:1787695791.799143:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96cc0. 00000100:00000001:1.0:1787695791.799148:0:2391:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:0.0:1787695791.799149:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695790/real 1787695791] req@ffff8b14465af800 x1874531139522944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695791.799162:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000200:1.0:1787695791.799164:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14465ac000 x1874531139523200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695815 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695791.799165:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787695791.799176:0:2391:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000400:00000010:0.0:1787695791.799178:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b7e8. 00000100:00000040:1.0:1787695791.799184:0:2391:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ac000 x1874531139523200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695815 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695791.799185:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000400:00000010:0.0:1787695791.799190:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000001:0.0:1787695791.799195:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787695791.799200:0:2391:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695791.799204:0:2391:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000200:0.0:1787695791.799205:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465af800 x1874531139522944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1787695791.799218:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000100:00000200:0.0:1787695791.799224:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465af800 x1874531139522944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:1.0:1787695791.799225:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1445678300. 00000400:00000200:1.0:1787695791.799233:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:0.0:1787695791.799241:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:1.0:1787695791.799242:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000100:00000001:0.0:1787695791.799245:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1787695791.799248:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96dd0 00000100:00000001:0.0:1787695791.799249:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695791.799253:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:1.0:1787695791.799254:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96dd0. 00000100:00000001:1.0:1787695791.799260:0:2391:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:0.0:1787695791.799262:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465af800 x1874531139522944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695791.799268:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14465aed80 x1874531139523328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695805 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.799278:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695791.799288:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465af800 x1874531139522944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695791.799288:0:2391:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695791.799297:0:2391:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465aed80 x1874531139523328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695805 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.799303:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695791.799309:0:2391:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695791.799311:0:2391:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695791.799324:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.799326:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695791.799328:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.799330:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695791.799332:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695791.799335:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695791.799340:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695791.799343:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695791.799347:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465af800 x1874531139522944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695791.799355:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139522944:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695791.799400:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695791.799405:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695790/real 1787695791] req@ffff8b14465ad880 x1874531139522688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695791.799415:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695791.799417:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695791.799418:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b9a0. 00000400:00000200:0.0:1787695791.799422:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96330 00000400:00000010:0.0:1787695791.799423:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96330. 00000100:00000001:0.0:1787695791.799426:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695791.799429:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ad880 x1874531139522688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695791.799438:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ad880 x1874531139522688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.799445:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695791.799447:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.799449:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695791.799451:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695791.799454:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ad880 x1874531139522688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.799462:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695791.799466:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ad880 x1874531139522688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.799473:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695791.799476:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.799478:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695791.799479:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.799482:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695791.799484:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695791.799487:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695791.799498:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695791.799501:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695791.799506:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ad880 x1874531139522688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695791.799515:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139522688:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695791.799520:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695791.799523:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695791.799526:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ace00 x1874531139522816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695805 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.799533:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695791.799536:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695791.799541:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14475fbc00. 02000000:00000001:0.0:1787695791.799546:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695791.799548:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695791.799551:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=24 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695791.799554:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695791.799556:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661800. 02000000:00000001:0.0:1787695791.799560:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695791.799562:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695791.799564:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ace00. 00000100:00000001:0.0:1787695791.799568:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695791.799569:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.799571:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695791.799574:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465af800 x1874531139522944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.799579:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695791.799581:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695791.799583:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14475f9c00. 02000000:00000001:0.0:1787695791.799586:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695791.799587:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695791.799588:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:0.0:1787695791.799590:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695791.799600:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445662400. 02000000:00000001:0.0:1787695791.799603:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695791.799604:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695791.799605:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465af800. 00000100:00000001:0.0:1787695791.799609:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695791.799610:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.799611:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695791.799615:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ad880 x1874531139522688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.799622:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695791.799623:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695791.799624:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14475f9800. 02000000:00000001:0.0:1787695791.799627:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695791.799628:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695791.799629:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695791.799631:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695791.799633:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445660000. 02000000:00000001:0.0:1787695791.799635:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695791.799636:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695791.799637:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ad880. 00000100:00000001:0.0:1787695791.799640:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695791.799641:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.799643:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695791.799645:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695791.799647:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695791.799649:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695791.799653:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695790/real 1787695791] req@ffff8b14465adf80 x1874531139523072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695791.799663:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695791.799664:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695791.799666:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b8f0. 00000400:00000200:0.0:1787695791.799669:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f965d8 00000400:00000010:0.0:1787695791.799670:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f965d8. 00000100:00000001:0.0:1787695791.799673:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695791.799676:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465adf80 x1874531139523072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695791.799699:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465adf80 x1874531139523072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.799706:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695791.799708:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.799710:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695791.799712:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695791.799716:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465adf80 x1874531139523072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.799722:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695791.799727:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465adf80 x1874531139523072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.799734:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695791.799737:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.799739:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695791.799740:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.799743:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695791.799744:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695791.799747:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695791.799751:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695791.799754:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695791.799759:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465adf80 x1874531139523072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695791.799767:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139523072:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695791.799772:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695791.799776:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695790/real 1787695791] req@ffff8b14465ac000 x1874531139523200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695791.799797:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695791.799798:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695791.799800:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617ba50. 00000400:00000200:0.0:1787695791.799803:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96110 00000400:00000010:0.0:1787695791.799805:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96110. 00000100:00000001:0.0:1787695791.799808:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695791.799812:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ac000 x1874531139523200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695791.799821:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ac000 x1874531139523200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.799827:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695791.799828:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.799830:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695791.799832:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695791.799835:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ac000 x1874531139523200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.799842:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695791.799847:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ac000 x1874531139523200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.799854:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695791.799856:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.799858:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695791.799859:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.799862:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695791.799864:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695791.799866:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695791.799870:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695791.799872:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695791.799883:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ac000 x1874531139523200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695791.799891:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139523200:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695791.799896:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695791.799899:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695790/real 1787695791] req@ffff8b14465aed80 x1874531139523328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695791.799907:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695791.799909:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695791.799911:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b528. 00000400:00000200:0.0:1787695791.799914:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96770 00000400:00000010:0.0:1787695791.799915:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96770. 00000100:00000001:0.0:1787695791.799918:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695791.799920:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465aed80 x1874531139523328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695791.799928:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465aed80 x1874531139523328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.799934:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695791.799936:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.799938:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695791.799939:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695791.799943:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465aed80 x1874531139523328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.799949:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695791.799952:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465aed80 x1874531139523328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695805 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.799959:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695791.799961:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.799967:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695791.799969:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.799971:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695791.799973:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695791.799975:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695791.799979:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695791.799982:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695791.799986:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465aed80 x1874531139523328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695805 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695791.799994:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139523328:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695791.799999:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695791.800001:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695791.800003:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465adf80 x1874531139523072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.800010:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695791.800025:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695791.800027:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14475f8800. 02000000:00000001:0.0:1787695791.800033:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695791.800034:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695791.800036:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:0.0:1787695791.800038:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695791.800040:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663c00. 02000000:00000001:0.0:1787695791.800043:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695791.800044:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695791.800045:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465adf80. 00000100:00000001:0.0:1787695791.800048:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695791.800049:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.800051:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695791.800054:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ac000 x1874531139523200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695815 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.800061:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695791.800068:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695791.800070:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445435c00. 02000000:00000001:0.0:1787695791.800073:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695791.800075:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695791.800076:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695791.800078:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695791.800079:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445662c00. 02000000:00000001:0.0:1787695791.800082:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695791.800083:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695791.800085:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ac000. 00000100:00000001:0.0:1787695791.800087:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695791.800088:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.800090:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695791.800094:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465aed80 x1874531139523328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695805 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695791.800101:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695791.800102:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695791.800104:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14514b6c00. 02000000:00000001:0.0:1787695791.800107:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695791.800108:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695791.800110:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=25 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695791.800112:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695791.800113:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663800. 02000000:00000001:0.0:1787695791.800115:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695791.800117:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695791.800118:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465aed80. 00000100:00000001:0.0:1787695791.800121:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695791.800122:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.800125:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695791.800126:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695791.800128:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695791.800130:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695791.800132:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695791.974454:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695791.974470:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695791.974471:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695791.974476:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695791.974490:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695791.974494:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695791.974496:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695792.054415:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695792.054422:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695792.054424:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695792.054429:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695792.054432:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695792.054435:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695792.054437:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695792.054439:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695792.054441:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695792.502438:0:41120:0:(connection.c:75:cpu_latency_work()) work item of ffff8b1460486220 (cpu 2) has reached its deadline 4298352581, at 4298352640 00000100:00000040:3.0:1787695792.502440:0:41781:0:(connection.c:75:cpu_latency_work()) work item of ffff8b1460486330 (cpu 3) has reached its deadline 4298352581, at 4298352640 00000100:00000010:3.0:1787695792.502453:0:41781:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8b1446635120. 00000100:00000010:2.0:1787695792.502459:0:41120:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8b1446635cc0. 00010000:00000001:1.0:1787695792.694471:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695792.694478:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695792.694485:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695792.694489:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695792.694491:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695792.694495:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695792.694498:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695792.694501:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695792.694503:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695792.694505:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695792.694509:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695792.694511:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695792.694535:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695792.694536:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695792.694540:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695792.694542:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695792.694545:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695792.694546:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695792.694604:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695792.694607:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695792.694610:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787695793.014438:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695793.014450:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695793.014453:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695793.014458:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695793.014460:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695793.014464:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695793.014467:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695793.078471:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695793.078480:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695793.078482:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695793.078488:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695793.078492:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695793.078496:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695793.078517:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695793.078520:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695793.078523:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695793.718527:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695793.718537:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695793.718539:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695793.718543:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695793.718548:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695793.718551:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695793.718552:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695793.718554:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695793.718558:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695793.718559:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695793.718562:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695793.718563:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695793.718566:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695793.718567:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695793.718570:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695793.718571:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695793.718574:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695793.718575:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695793.718652:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695793.718657:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695793.718661:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787695794.038506:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695794.038524:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695794.038525:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695794.038530:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695794.038532:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695794.038536:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695794.038538:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695794.102488:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695794.102496:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695794.102499:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695794.102505:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695794.102509:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695794.102512:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695794.102515:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695794.102518:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695794.102521:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695794.742543:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695794.742552:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695794.742560:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695794.742561:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695794.742566:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695794.742567:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695794.742572:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695794.742573:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695794.742578:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695794.742579:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695794.742583:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695794.742584:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695794.742653:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695794.742659:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695794.742663:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787695795.062475:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695795.062490:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695795.062493:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695795.062499:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695795.062501:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695795.062504:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695795.062507:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695795.126459:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695795.126466:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695795.126468:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695795.126472:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695795.126474:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695795.126478:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695795.126480:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695795.126482:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695795.126484:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695795.766473:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695795.766480:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695795.766502:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695795.766504:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695795.766507:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695795.766509:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695795.766512:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695795.766514:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695795.766517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695795.766518:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695795.766521:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695795.766523:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695795.766568:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695795.766572:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695795.766574:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787695796.086436:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695796.086450:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695796.086453:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695796.086458:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695796.086461:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695796.086464:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695796.086467:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695796.086529:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695796.086536:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1787695796.086538:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695796.086541:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695796.086544:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695796.086547:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695796.086551:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695796.086558:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3683 00000400:00000200:3.0:1787695796.086567:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695796.086574:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695796.086589:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695796.086593:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695796.086597:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695796.086600:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695796.086603:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695796.086607:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695796.086609:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:3.0:1787695796.086612:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695796.086615:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:3.0:1787695796.086618:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:3.0:1787695796.086621:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:3.0:1787695796.086624:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695796.086626:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695796.086629:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695796.086631:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695796.086634:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:3.0:1787695796.086644:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695796.086646:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695796.086649:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695796.086657:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695796.086660:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695796.086662:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695796.086667:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695796.086670:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695796.086673:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695796.086680:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6d80. 00000020:00000040:3.0:1787695796.086684:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695796.086688:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695796.086690:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695796.086693:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695796.086695:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695796.086709:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695796.086717:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f4400. 00000100:00000001:3.0:1787695796.086724:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695796.086733:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6d80 x1874531139523456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695796.086741:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695796.086744:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695796.086747:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c3be57b] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6d80 x1874531139523456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695796.086765:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695796.086768:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695796.086770:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695796.086771:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695796.086773:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695796.086775:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695796.086776:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3683 00000400:00000200:3.0:1787695796.086780:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695796.086783:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00001000:3.0:1787695796.086786:0:38:0:(import.c:2149:obd_at_measure()) add 15 to ffff8b1458081c18 time=112 v=10 (10 0 0 0) 00000100:00001000:3.0:1787695796.086790:0:38:0:(import.c:2215:obd_at_measure()) AT ffff8b1458081c18 change: old=10 new=15 delta=5 (val=15) hist 15 0 0 0 00000100:00080000:3.0:1787695796.086792:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 10s 00000100:00000001:0.0:1787695796.086794:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1787695796.086796:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695796.086798:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695796.086799:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695796.086802:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1787695796.086803:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695796.086806:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695796.086807:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695796.086808:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:3.0:1787695796.086812:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695796.086813:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:3.0:1787695796.086816:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:0.0:1787695796.086818:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6d80 x1874531139523456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695796.086829:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000001:0.0:1787695796.086831:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695796.086833:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695796.086838:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1787695796.086840:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695796.086841:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695796.086846:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695796.086847:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:0.0:1787695796.086852:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139523456:192.168.201.140@tcp:38:kworker.0 00000100:00000040:3.0:1787695796.086855:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695796.086860:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695796.086862:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:3.0:1787695796.086866:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695796.086871:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695796.086873:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787695796.086875:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787695796.086878:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695796.086881:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695796.086882:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787695796.086883:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787695796.086887:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787695796.086887:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695796.086894:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695796.086895:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787695796.086895:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695796.086902:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5180. 02000000:00000010:0.0:1787695796.086905:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14514b4400. 00000020:00000040:3.0:1787695796.086905:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=25 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695796.086911:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787695796.086912:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695796.086913:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695796.086916:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695796.086918:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695796.086919:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695796.086923:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000010:3.0:1787695796.086927:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f5400. 00000400:00000010:0.0:1787695796.086929:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b528. 00000100:00000001:3.0:1787695796.086932:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695796.086949:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96770. 00000100:00100000:3.0:1787695796.086954:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5180 x1874531139523584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1787695796.086959:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139523456, portal 10 00000100:00000001:3.0:1787695796.086964:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695796.086966:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787695796.086970:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695796.086971:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930155392 : -128555779396224 : ffff8b14466f6d80) 00000100:00000040:3.0:1787695796.086979:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a115abf4] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5180 x1874531139523584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695796.086987:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6d80 x1874531139523456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695796.086995:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695796.087000:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695796.087006:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00080000:3.0:1787695796.087007:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:0.0:1787695796.087010:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96110. 00000100:00080000:3.0:1787695796.087014:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000200:0.0:1787695796.087016:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139523456, offset 0 00000100:00000001:3.0:1787695796.087018:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695796.087022:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1787695796.087025:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695796.087029:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695796.087036:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3683 00000400:00000200:0.0:1787695796.087037:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695796.087047:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695796.087048:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:3.0:1787695796.087054:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695796.087056:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1787695796.087058:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1787695796.087061:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695796.087064:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695796.087066:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695796.087068:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695796.087069:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695796.087070:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:3.0:1787695796.087071:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695796.087073:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:3.0:1787695796.087074:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:3.0:1787695796.087075:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000400:00000200:0.0:1787695796.087078:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000040:3.0:1787695796.087079:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695796.087081:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695796.087082:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695796.087087:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695796.087088:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000400:00000200:0.0:1787695796.087092:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00080000:3.0:1787695796.087093:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695796.087095:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695796.087096:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695796.087099:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695796.087103:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695796.087105:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:0.0:1787695796.087107:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695796.087115:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16899:16899:256:4294967295] 192.168.201.140@tcp LPNI seq info [16898:16898:8:4294967295] 00000008:00100000:3.0:1787695796.087116:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787695796.087121:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695796.087126:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695796.087130:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000010:3.0:1787695796.087130:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4380. 00000020:00000040:3.0:1787695796.087135:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:3.0:1787695796.087137:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695796.087138:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695796.087140:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695796.087141:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695796.087143:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695796.087149:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f5000. 00000400:00000200:0.0:1787695796.087150:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:3.0:1787695796.087153:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1787695796.087159:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00100000:3.0:1787695796.087161:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4380 x1874531139523712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:0.0:1787695796.087166:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1445678300. 00000100:00000001:3.0:1787695796.087170:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695796.087173:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1787695796.087177:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:3.0:1787695796.087178:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d2982c15] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4380 x1874531139523712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1787695796.087186:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787695796.087193:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695796.087196:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695796.087198:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1787695796.087200:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695796.087202:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695796.087205:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695796.087206:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:0.0:1787695796.087213:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000100:00080000:3.0:1787695796.087213:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3683 00000400:00000200:0.0:1787695796.087223:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695796.087225:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695796.087229:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298357701 00000400:00000200:3.0:1787695796.087234:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000010:0.0:1787695796.087238:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8b1446635720. 00000100:00080000:3.0:1787695796.087239:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695796.087246:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695796.087248:0:2402:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request now active for cpu 2 00000100:00000001:3.0:1787695796.087251:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695796.087256:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695796.087257:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298357701 00000100:00000010:0.0:1787695796.087263:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8b1446635d80. 00000100:00000001:3.0:1787695796.087265:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695796.087268:0:2402:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request now active for cpu 3 00000020:00000001:3.0:1787695796.087269:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695796.087271:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000100:00000001:0.0:1787695796.087274:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1787695796.087274:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695796.087278:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000100:00000001:0.0:1787695796.087279:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695796.087279:0:2393:0:(lib-socket.c:438:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.140:988 00000020:00000001:3.0:1787695796.087282:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000001:0.0:1787695796.087285:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695796.087287:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000001:0.0:1787695796.087289:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695796.087292:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695796.087296:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695796.087298:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:3.0:1787695796.087298:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695796.087303:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695796.087303:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695796.087309:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00000001:0.0:1787695796.087311:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787695796.087314:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695796.087316:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695796.087320:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787695796.087321:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695796.087325:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695796.087329:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1787695796.087330:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5180 x1874531139523584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695796.087335:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695796.087339:0:2393:0:(lib-socket.c:438:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.140:988 00000100:00000001:3.0:1787695796.087341:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695796.087347:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787695796.087349:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695796.087352:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695796.087355:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:3.0:1787695796.087355:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695796.087359:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695796.087360:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695796.087391:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695796.087407:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139523584:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695796.087411:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695796.087414:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695796.087416:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695796.087418:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695796.087421:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14514b5c00. 02000000:00000001:0.0:1787695796.087422:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695796.087427:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695796.087429:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617ba50. 00000400:00000010:0.0:1787695796.087433:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f965d8. 00000100:00000200:0.0:1787695796.087435:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139523584, portal 10 00000100:00000001:0.0:1787695796.087437:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695796.087438:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930148224 : -128555779403392 : ffff8b14466f5180) 00000100:00000040:0.0:1787695796.087445:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5180 x1874531139523584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695796.087452:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000010:3.0:1787695796.087452:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5500. 00000100:00000040:0.0:1787695796.087458:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000020:00000040:3.0:1787695796.087460:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 00000400:00000010:0.0:1787695796.087461:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96330. 00000100:00000200:0.0:1787695796.087466:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139523584, offset 0 02000000:00000001:3.0:1787695796.087466:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695796.087469:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695796.087472:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:0.0:1787695796.087473:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695796.087474:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695796.087479:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695796.087482:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000010:3.0:1787695796.087486:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f7400. 00000400:00000200:0.0:1787695796.087487:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695796.087493:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1787695796.087493:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695796.087498:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695796.087502:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00100000:3.0:1787695796.087503:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5500 x1874531139523840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1787695796.087506:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695796.087512:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16900:16900:255:4294967295] 192.168.201.140@tcp LPNI seq info [16899:16899:7:4294967295] 00000400:00000200:0.0:1787695796.087519:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695796.087529:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:3.0:1787695796.087537:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:0.0:1787695796.087538:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695796.087541:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695796.087545:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a777d68] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5500 x1874531139523840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:0.0:1787695796.087550:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1445d7af00. 00000100:00000001:3.0:1787695796.087558:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695796.087561:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695796.087565:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695796.087566:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:0.0:1787695796.087567:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00080000:3.0:1787695796.087569:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695796.087576:0:38:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:0.1:1787695796.087577:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695796.087581:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695796.087582:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3683 00000100:00000040:0.0:1787695796.087585:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:3.0:1787695796.087591:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695796.087592:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695796.087597:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298356550 00000100:00000001:0.0:1787695796.087600:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298357701 00000400:00000200:3.0:1787695796.087602:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:0.0:1787695796.087603:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298356550 00000100:00080000:3.0:1787695796.087609:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787695796.087610:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298357701 00000100:00000001:0.0:1787695796.087613:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695796.087616:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695796.087620:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1787695796.087620:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695796.087624:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695796.087627:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4380 x1874531139523712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695796.087629:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695796.087634:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695796.087640:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787695796.087641:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1787695796.087642:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000100:00000001:0.0:1787695796.087644:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695796.087645:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1787695796.087648:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:3.0:1787695796.087649:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 02000000:00000001:0.0:1787695796.087651:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1787695796.087653:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 02000000:00000001:0.0:1787695796.087654:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695796.087657:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00100000:0.0:1787695796.087661:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139523712:192.168.201.140@tcp:8:kworker.0 00000100:00000040:3.0:1787695796.087665:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695796.087666:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695796.087669:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787695796.087669:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787695796.087673:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695796.087673:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695796.087678:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787695796.087678:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000010:0.0:1787695796.087681:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14514b6c00. 02000000:00000001:0.0:1787695796.087683:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695796.087687:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000040:3.0:1787695796.087688:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000400:00000010:0.0:1787695796.087691:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b8f0. 00000100:00080000:3.0:1787695796.087693:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000010:0.0:1787695796.087696:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00000001:3.0:1787695796.087696:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695796.087701:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:0.0:1787695796.087703:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139523712, portal 4 02000000:00000001:3.0:1787695796.087705:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695796.087706:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1787695796.087708:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695796.087710:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695796.087712:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787695796.087716:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930144640 : -128555779406976 : ffff8b14466f4380) 00000008:00000001:3.0:1787695796.087716:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695796.087721:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695796.087725:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4700. 00000100:00000040:0.0:1787695796.087727:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4380 x1874531139523712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1787695796.087729:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:3.0:1787695796.087734:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695796.087737:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695796.087740:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787695796.087744:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787695796.087744:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695796.087749:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695796.087750:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 02000000:00000010:3.0:1787695796.087754:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f5c00. 00000400:00000010:0.0:1787695796.087755:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96b28. 00000100:00000001:3.0:1787695796.087759:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695796.087760:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139523712, offset 0 00000400:00000200:0.0:1787695796.087769:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00100000:3.0:1787695796.087769:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4700 x1874531139523968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1787695796.087777:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695796.087784:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:3.0:1787695796.087785:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1787695796.087789:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1787695796.087789:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695796.087795:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000040:3.0:1787695796.087799:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008818c2cc] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4700 x1874531139523968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695796.087801:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695796.087807:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695796.087814:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16901:16901:254:4294967295] 192.168.201.140@tcp LPNI seq info [16900:16900:6:4294967295] 00000100:00000001:3.0:1787695796.087821:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695796.087822:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:3.0:1787695796.087828:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1787695796.087829:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00080000:3.0:1787695796.087837:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000200:0.0:1787695796.087838:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695796.087843:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000010:0.0:1787695796.087844:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1445d7a100. 00000100:00080000:3.0:1787695796.087849:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:0.0:1787695796.087850:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787695796.087857:0:38:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:0.1:1787695796.087858:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695796.087862:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695796.087865:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:0.0:1787695796.087873:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:3.0:1787695796.087873:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3683 00000100:00000001:0.0:1787695796.087880:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298356580 00000400:00000200:3.0:1787695796.087883:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695796.087886:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298357701 00000400:00000200:3.0:1787695796.087888:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:0.0:1787695796.087893:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298356580 00000100:00001000:3.0:1787695796.087894:0:38:0:(import.c:2149:obd_at_measure()) add 15 to ffff8b144515c418 time=112 v=10 (10 0 0 0) 00000100:00000001:0.0:1787695796.087900:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298357701 00000100:00001000:3.0:1787695796.087901:0:38:0:(import.c:2215:obd_at_measure()) AT ffff8b144515c418 change: old=10 new=15 delta=5 (val=15) hist 15 0 0 0 00000100:00000001:0.0:1787695796.087907:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1787695796.087908:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 10s 00000100:00000001:0.0:1787695796.087911:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695796.087914:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695796.087916:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695796.087917:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695796.087919:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695796.087920:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695796.087922:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695796.087924:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000100:00000001:0.0:1787695796.087926:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787695796.087927:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695796.087930:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000100:00000001:0.0:1787695796.087933:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:3.0:1787695796.087933:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000020:00000001:3.0:1787695796.087934:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:0.0:1787695796.087949:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695796.087953:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695796.087955:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695796.087956:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695796.087959:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1787695796.087960:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695796.087964:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787695796.087964:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695796.087968:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00080000:3.0:1787695796.087974:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000040:0.0:1787695796.087976:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5500 x1874531139523840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695796.087976:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695796.087980:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695796.087984:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695796.087988:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695796.087991:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787695796.087995:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1787695796.087995:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787695796.087998:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787695796.087999:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695796.088001:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787695796.088003:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695796.088004:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1787695796.088008:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5f80. 02000000:00000001:0.0:1787695796.088009:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695796.088012:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139523840:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695796.088018:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1787695796.088019:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=26 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:0.0:1787695796.088020:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695796.088024:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695796.088024:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787695796.088027:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787695796.088028:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695796.088030:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695796.088031:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000010:0.0:1787695796.088033:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445435c00. 02000000:00000001:3.0:1787695796.088034:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695796.088037:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695796.088041:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f5800. 00000400:00000200:0.0:1787695796.088042:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695796.088047:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695796.088049:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b9a0. 00000100:00100000:3.0:1787695796.088054:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5f80 x1874531139524096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:0.0:1787695796.088057:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96dd0. 00000100:00000200:0.0:1787695796.088064:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139523840, portal 4 00000100:00000001:3.0:1787695796.088065:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695796.088068:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695796.088070:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:3.0:1787695796.088073:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006e7bde76] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5f80 x1874531139524096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695796.088074:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930149120 : -128555779402496 : ffff8b14466f5500) 00000100:00000001:3.0:1787695796.088085:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695796.088088:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5500 x1874531139523840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695796.088088:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787695796.088096:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695796.088101:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787695796.088109:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1787695796.088111:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695796.088115:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695796.088116:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3693) 00000400:00000010:0.0:1787695796.088118:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96cc0. 00000100:00000200:0.0:1787695796.088123:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139523840, offset 0 00000400:00000200:0.0:1787695796.088126:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695796.088129:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695796.088137:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695796.088139:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695796.088141:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695796.088143:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695796.088145:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695796.088147:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16902:16902:253:4294967295] 192.168.201.140@tcp LPNI seq info [16901:16901:5:4294967295] 00000400:00000200:0.0:1787695796.088149:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695796.088154:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695796.088157:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695796.088160:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1445d7aa00. 00000800:00000200:0.0:1787695796.088162:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695796.088165:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695796.088167:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695796.088169:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:0.0:1787695796.088171:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695796.088173:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298356252 00000100:00000001:0.0:1787695796.088174:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298357701 00000100:00000001:0.0:1787695796.088176:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298356252 00000100:00000001:0.0:1787695796.088178:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298357701 00000100:00000001:0.0:1787695796.088179:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695796.088180:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695796.088182:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695796.088185:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4700 x1874531139523968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695796.088189:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695796.088190:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695796.088192:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695796.088193:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695796.088195:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695796.088196:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139523968:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695796.088203:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695796.088204:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695796.088205:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695796.088207:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695796.088210:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14475f8800. 02000000:00000001:0.0:1787695796.088213:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695796.088216:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695796.088219:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b7e8. 00000400:00000010:0.0:1787695796.088223:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96f68. 00000100:00000200:0.0:1787695796.088226:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139523968, portal 10 00000100:00000001:0.0:1787695796.088228:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695796.088229:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930145536 : -128555779406080 : ffff8b14466f4700) 00000100:00000040:0.0:1787695796.088236:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4700 x1874531139523968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695796.088243:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695796.088246:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695796.088248:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96908. 00000100:00000200:0.0:1787695796.088251:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139523968, offset 0 00000400:00000200:0.0:1787695796.088256:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695796.088260:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695796.088263:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695796.088265:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695796.088267:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695796.088269:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695796.088270:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695796.088272:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16903:16903:252:4294967295] 192.168.201.140@tcp LPNI seq info [16902:16902:4:4294967295] 00000400:00000200:0.0:1787695796.088275:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695796.088280:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695796.088283:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695796.088285:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1445d7a200. 00000800:00000200:0.0:1787695796.088289:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695796.088298:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695796.088301:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695796.088305:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:0.0:1787695796.088309:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695796.088311:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298356438 00000100:00000001:0.0:1787695796.088313:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298357701 00000100:00000001:0.0:1787695796.088316:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298356438 00000100:00000001:0.0:1787695796.088318:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298357701 00000100:00000001:0.0:1787695796.088320:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695796.088322:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695796.088325:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695796.088328:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695796.088331:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695796.088332:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695796.088334:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695796.088335:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695796.088336:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695796.088339:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5f80 x1874531139524096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695796.088344:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695796.088345:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695796.088347:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695796.088348:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695796.088349:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695796.088351:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139524096:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695796.088353:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695796.088355:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695796.088356:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695796.088357:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695796.088359:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14475f9800. 02000000:00000001:0.0:1787695796.088361:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695796.088382:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695796.088385:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b6e0. 00000400:00000010:0.0:1787695796.088395:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96198. 00000100:00000200:0.0:1787695796.088399:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139524096, portal 10 00000100:00000001:0.0:1787695796.088402:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695796.088403:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930151808 : -128555779399808 : ffff8b14466f5f80) 00000100:00000040:0.0:1787695796.088410:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5f80 x1874531139524096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695796.088417:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695796.088419:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695796.088421:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96550. 00000100:00000200:0.0:1787695796.088423:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139524096, offset 0 00000400:00000200:0.0:1787695796.088426:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695796.088429:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695796.088431:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695796.088433:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695796.088435:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695796.088436:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695796.088438:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695796.088440:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16904:16904:251:4294967295] 192.168.201.140@tcp LPNI seq info [16903:16903:3:4294967295] 00000400:00000200:0.0:1787695796.088443:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695796.088447:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695796.088450:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695796.088452:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1445d7a400. 00000800:00000200:0.0:1787695796.088454:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695796.088457:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695796.088459:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695796.088461:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:0.0:1787695796.088463:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695796.088464:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298356417 00000100:00000001:0.0:1787695796.088466:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298357701 00000100:00000001:0.0:1787695796.088467:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298356417 00000100:00000001:0.0:1787695796.088469:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298357701 00000100:00000001:0.0:1787695796.088474:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695796.088475:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695796.088477:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695796.088479:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695796.088481:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695796.088482:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695796.088484:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695796.088485:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695796.088486:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695796.088488:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695796.150500:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695796.150510:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695796.150512:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695796.150518:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695796.150522:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695796.150526:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695796.150528:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695796.150530:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695796.150532:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695796.790435:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695796.790441:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695796.790445:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695796.790446:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695796.790449:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695796.790454:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695796.790456:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695796.790462:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695796.790464:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695796.790465:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695796.790468:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695796.790470:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695796.790472:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695796.790474:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695796.790479:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695796.790481:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695796.790482:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695796.790483:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695796.790484:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695796.790485:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695796.790486:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695796.790487:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695796.790490:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695796.790493:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695796.790498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695796.790499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695796.790501:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695796.790502:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695796.790503:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695796.790520:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695796.790521:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695796.790522:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695796.790523:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695796.790524:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695796.790525:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695796.790526:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695796.790590:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695796.790597:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695796.790602:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1787695797.110492:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695797.110504:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695797.110506:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695797.110512:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695797.110515:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695797.110521:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695797.110524:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695797.174535:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695797.174546:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695797.174571:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695797.174592:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695797.174597:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695797.174602:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695797.174605:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695797.174609:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695797.174612:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695797.622513:0:41120:0:(connection.c:75:cpu_latency_work()) work item of ffff8b1460486220 (cpu 2) has reached its deadline 4298357701, at 4298357760 00000100:00000040:3.0:1787695797.622513:0:41781:0:(connection.c:75:cpu_latency_work()) work item of ffff8b1460486330 (cpu 3) has reached its deadline 4298357701, at 4298357760 00000100:00000010:3.0:1787695797.622544:0:41781:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8b1446635d80. 00000100:00000010:2.0:1787695797.622553:0:41120:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8b1446635720. 00010000:00000001:1.0:1787695797.814484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695797.814491:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695797.814497:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695797.814499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695797.814503:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695797.814504:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695797.814508:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695797.814509:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695797.814512:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695797.814514:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695797.814517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695797.814518:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695797.814580:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695797.814586:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695797.814590:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787695797.942732:0:2393:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.201.140:988 00000400:00000100:2.0:1787695797.942783:0:2393:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.140@tcp at host 192.168.201.140 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1787695797.942795:0:2393:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695797.942800:0:2393:0:(router.c:1791:lnet_notify()) 192.168.201.40@tcp notifying 192.168.201.140@tcp: down 00000400:00000200:2.0:1787695797.942806:0:2393:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695797.942812:0:2393:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:2.0:1787695797.942818:0:2393:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:2.0:1787695797.942823:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1445678300. 00000400:00000200:2.0:1787695797.942832:0:2393:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1787695797.942838:0:2393:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1787695797.942852:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96110 00000400:00000010:2.0:1787695797.942856:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96110. 00000100:00000001:2.0:1787695797.942863:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695797.942890:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f6d80 x1874531139523456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695797.942955:0:2393:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695797.942961:0:2393:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6d80 x1874531139523456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695821 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695797.942969:0:2393:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695797.942972:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787695797.942985:0:2393:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:2.0:1787695797.942989:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1445d7af00. 00000400:00000200:2.0:1787695797.942994:0:2393:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1787695797.942998:0:2393:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1787695797.943002:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96330 00000400:00000010:2.0:1787695797.943004:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96330. 00000100:00000001:2.0:1787695797.943007:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787695797.943013:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:2.0:1787695797.943014:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f5180 x1874531139523584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.943021:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695797.943028:0:2393:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787695797.943029:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:2.0:1787695797.943034:0:2393:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5180 x1874531139523584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695816 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695797.943044:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695796/real 1787695797] req@ffff8b14466f5180 x1874531139523584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695816 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695797.943045:0:2393:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695797.943049:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787695797.943061:0:2393:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:2.0:1787695797.943065:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1445d7a100. 00000400:00000200:2.0:1787695797.943069:0:2393:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000001:1.0:1787695797.943070:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695797.943074:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1787695797.943075:0:2393:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000010:1.0:1787695797.943080:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617ba50. 00000400:00000200:2.0:1787695797.943081:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96b28 00000400:00000010:2.0:1787695797.943086:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96b28. 00000400:00000200:1.0:1787695797.943091:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f965d8 00000100:00000001:2.0:1787695797.943091:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000400:00000010:1.0:1787695797.943096:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f965d8. 00000100:00000200:2.0:1787695797.943101:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f4380 x1874531139523712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.943103:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695797.943113:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5180 x1874531139523584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695797.943116:0:2393:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000200:1.0:1787695797.943132:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5180 x1874531139523584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695797.943133:0:2393:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4380 x1874531139523712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695821 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.943157:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695797.943158:0:2393:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695797.943160:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787695797.943168:0:2393:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000100:00000001:1.0:1787695797.943171:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:2.0:1787695797.943173:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1445d7aa00. 00000100:00000001:1.0:1787695797.943178:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:2.0:1787695797.943180:0:2393:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:1.0:1787695797.943183:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1787695797.943188:0:2393:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1787695797.943194:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96cc0 00000100:00100000:1.0:1787695797.943195:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5180 x1874531139523584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1787695797.943198:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96cc0. 00000100:00000001:2.0:1787695797.943206:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787695797.943210:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1787695797.943215:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f5500 x1874531139523840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695797.943222:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5180 x1874531139523584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695816 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695797.943234:0:2393:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787695797.943240:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:2.0:1787695797.943243:0:2393:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5500 x1874531139523840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695821 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.943247:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.943252:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695797.943255:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695797.943259:0:2393:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695797.943263:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695797.943264:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787695797.943272:0:2393:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:2.0:1787695797.943274:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1445d7a200. 00000100:00000001:1.0:1787695797.943275:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:2.0:1787695797.943278:0:2393:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00080000:1.0:1787695797.943283:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000400:00000200:2.0:1787695797.943285:0:2393:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1787695797.943292:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96908 00000100:00080000:1.0:1787695797.943295:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000400:00000010:2.0:1787695797.943296:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96908. 00000100:00000001:1.0:1787695797.943302:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1787695797.943303:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695797.943312:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f4700 x1874531139523968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695821 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695797.943313:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5180 x1874531139523584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695816 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695797.943340:0:2393:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:1.0:1787695797.943343:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139523584:192.168.201.140@tcp:38:kworker.0 00000100:00000040:2.0:1787695797.943349:0:2393:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4700 x1874531139523968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695821 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.943353:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787695797.943359:0:2393:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695797.943361:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695796/real 1787695797] req@ffff8b14466f4380 x1874531139523712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695797.943423:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787695797.943434:0:2393:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:2.0:1787695797.943442:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1445d7a400. 00000400:00000200:2.0:1787695797.943448:0:2393:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000001:1.0:1787695797.943450:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695797.943454:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695797.943459:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b8f0. 00000400:00000200:2.0:1787695797.943459:0:2393:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1787695797.943466:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96550 00000400:00000200:1.0:1787695797.943467:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000400:00000010:2.0:1787695797.943471:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96550. 00000400:00000010:1.0:1787695797.943472:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000001:2.0:1787695797.943477:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787695797.943485:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695797.943486:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f5f80 x1874531139524096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695816 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695797.943494:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4380 x1874531139523712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695797.943502:0:2393:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695797.943509:0:2393:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5f80 x1874531139524096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695816 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695797.943519:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4380 x1874531139523712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695797.943519:0:2393:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695797.943522:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695797.943533:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695797.943536:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.943538:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695797.943541:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695797.943546:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4380 x1874531139523712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.943554:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695797.943559:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4380 x1874531139523712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.943567:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695797.943570:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.943573:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695797.943574:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.943577:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695797.943579:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695797.943583:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695797.943589:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695797.943592:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695797.943597:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4380 x1874531139523712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695797.943607:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139523712:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695797.943615:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695797.943628:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695796/real 1787695797] req@ffff8b14466f6d80 x1874531139523456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695797.943644:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695797.943646:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695797.943648:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b528. 00000400:00000200:1.0:1787695797.943651:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96770 00000400:00000010:1.0:1787695797.943653:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96770. 00000100:00000001:1.0:1787695797.943656:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695797.943660:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6d80 x1874531139523456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695797.943669:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6d80 x1874531139523456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.943677:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695797.943678:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.943680:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695797.943682:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695797.943690:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6d80 x1874531139523456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.943698:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695797.943703:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6d80 x1874531139523456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.943711:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695797.943714:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.943716:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695797.943722:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.943725:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695797.943727:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695797.943730:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695797.943736:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695797.943745:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695797.943751:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6d80 x1874531139523456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695797.943760:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139523456:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695797.943766:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695797.943769:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695797.943773:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5180 x1874531139523584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695816 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.943781:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695797.943785:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695797.943792:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14514b5c00. 02000000:00000001:1.0:1787695797.943798:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695797.943801:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695797.943804:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=24 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695797.943807:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695797.943810:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f5400. 02000000:00000001:1.0:1787695797.943817:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695797.943819:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695797.943822:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5180. 00000100:00000001:1.0:1787695797.943827:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695797.943828:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.943831:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695797.943835:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4380 x1874531139523712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.943843:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695797.943844:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695797.943846:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14514b6c00. 02000000:00000001:1.0:1787695797.943850:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695797.943852:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695797.943853:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695797.943856:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695797.943857:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f5000. 02000000:00000001:1.0:1787695797.943867:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695797.943874:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695797.943875:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4380. 00000100:00000001:1.0:1787695797.943879:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695797.943880:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.943882:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695797.943887:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6d80 x1874531139523456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.943894:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695797.943895:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695797.943897:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14514b4400. 02000000:00000001:1.0:1787695797.943900:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695797.943901:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695797.943903:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695797.943905:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695797.943918:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f4400. 02000000:00000001:1.0:1787695797.943922:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695797.943923:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695797.943925:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6d80. 00000100:00000001:1.0:1787695797.943928:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695797.943929:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.943932:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695797.943934:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695797.943936:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695797.943938:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695797.943942:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695796/real 1787695797] req@ffff8b14466f5f80 x1874531139524096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695797.943953:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695797.943956:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695797.943958:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b6e0. 00000400:00000200:1.0:1787695797.943960:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96198 00000400:00000010:1.0:1787695797.943963:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96198. 00000100:00000001:1.0:1787695797.943966:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695797.943969:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5f80 x1874531139524096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695797.943992:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5f80 x1874531139524096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.943998:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695797.944000:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.944002:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695797.944004:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695797.944008:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5f80 x1874531139524096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.944016:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695797.944021:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5f80 x1874531139524096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695816 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.944029:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695797.944032:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.944034:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695797.944035:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.944037:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695797.944040:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695797.944042:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695797.944047:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695797.944049:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695797.944054:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5f80 x1874531139524096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695816 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695797.944062:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139524096:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695797.944067:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695797.944071:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695796/real 1787695797] req@ffff8b14466f5500 x1874531139523840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695797.944092:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695797.944093:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695797.944096:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b9a0. 00000400:00000200:1.0:1787695797.944099:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96dd0 00000400:00000010:1.0:1787695797.944100:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96dd0. 00000100:00000001:1.0:1787695797.944103:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695797.944107:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5500 x1874531139523840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695797.944115:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5500 x1874531139523840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.944122:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695797.944124:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.944126:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695797.944128:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695797.944132:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5500 x1874531139523840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.944140:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695797.944144:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5500 x1874531139523840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.944151:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695797.944154:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.944155:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695797.944157:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.944159:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695797.944161:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695797.944164:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695797.944168:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695797.944171:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695797.944184:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5500 x1874531139523840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695797.944192:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139523840:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695797.944197:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695797.944201:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695796/real 1787695797] req@ffff8b14466f4700 x1874531139523968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695797.944210:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695797.944212:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695797.944214:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b7e8. 00000400:00000200:1.0:1787695797.944217:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96f68 00000400:00000010:1.0:1787695797.944219:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96f68. 00000100:00000001:1.0:1787695797.944222:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695797.944226:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4700 x1874531139523968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695797.944236:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4700 x1874531139523968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.944245:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695797.944247:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.944250:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695797.944251:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695797.944256:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4700 x1874531139523968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.944264:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695797.944268:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4700 x1874531139523968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.944282:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695797.944284:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.944294:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695797.944295:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.944297:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695797.944300:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695797.944302:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695797.944307:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695797.944310:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695797.944315:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4700 x1874531139523968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695797.944323:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139523968:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695797.944328:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695797.944330:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695797.944333:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5f80 x1874531139524096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695816 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.944340:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695797.944341:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695797.944343:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14475f9800. 02000000:00000001:1.0:1787695797.944349:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695797.944351:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695797.944352:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=25 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695797.944355:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695797.944358:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f5800. 02000000:00000001:1.0:1787695797.944361:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695797.944412:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695797.944414:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5f80. 00000100:00000001:1.0:1787695797.944418:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695797.944419:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.944421:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695797.944424:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5500 x1874531139523840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.944431:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695797.944439:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695797.944441:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445435c00. 02000000:00000001:1.0:1787695797.944444:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695797.944446:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695797.944447:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695797.944449:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695797.944451:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f7400. 02000000:00000001:1.0:1787695797.944454:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695797.944455:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695797.944456:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5500. 00000100:00000001:1.0:1787695797.944459:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695797.944460:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.944462:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695797.944467:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4700 x1874531139523968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695821 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695797.944475:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695797.944476:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695797.944478:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14475f8800. 02000000:00000001:1.0:1787695797.944484:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695797.944485:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695797.944487:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695797.944489:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695797.944490:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f5c00. 02000000:00000001:1.0:1787695797.944493:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695797.944494:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695797.944496:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4700. 00000100:00000001:1.0:1787695797.944498:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695797.944499:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695797.944502:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695797.944504:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695797.944505:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695797.944507:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695797.944509:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695798.118489:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695798.118505:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695798.118507:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695798.118513:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695798.118537:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695798.118541:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695798.118543:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695798.198483:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695798.198489:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695798.198490:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695798.198496:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695798.198500:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695798.198504:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695798.198507:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695798.198510:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695798.198513:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695798.838492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695798.838502:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695798.838509:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695798.838510:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695798.838513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695798.838514:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695798.838518:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695798.838519:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695798.838521:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695798.838522:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695798.838525:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695798.838526:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695798.838593:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695798.838598:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695798.838602:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787695799.158450:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695799.158461:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695799.158462:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695799.158466:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695799.158469:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695799.158471:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695799.158473:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695799.222429:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695799.222437:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695799.222440:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695799.222447:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695799.222451:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695799.222454:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695799.222456:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695799.222457:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695799.222460:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695799.862438:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695799.862443:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695799.862447:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695799.862448:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695799.862451:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695799.862452:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695799.862455:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695799.862456:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695799.862458:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695799.862458:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695799.862461:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695799.862462:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695799.862515:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695799.862519:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695799.862521:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787695800.182489:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695800.182500:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695800.182502:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695800.182506:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695800.182508:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695800.182527:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695800.182529:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695800.246450:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695800.246459:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695800.246461:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695800.246467:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695800.246470:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695800.246474:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695800.246476:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695800.246503:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695800.246506:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695800.886548:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695800.886558:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695800.886567:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695800.886568:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695800.886573:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695800.886575:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695800.886580:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695800.886582:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695800.886586:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695800.886587:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695800.886591:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695800.886592:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695800.886661:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695800.886666:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695800.886670:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787695801.206492:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695801.206500:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:2.0:1787695801.206502:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1787695801.206510:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695801.206514:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787695801.206516:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695801.206518:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695801.206520:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695801.206523:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695801.206526:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787695801.206531:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3688 00000100:00000001:1.0:1787695801.206538:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1787695801.206543:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695801.206545:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1787695801.206550:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695801.206552:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695801.206555:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695801.206559:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695801.206560:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695801.206569:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695801.206574:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695801.206578:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695801.206588:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695801.206590:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:2.0:1787695801.206594:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695801.206597:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:2.0:1787695801.206600:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:2.0:1787695801.206602:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:2.0:1787695801.206605:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695801.206606:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695801.206609:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695801.206610:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695801.206612:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:2.0:1787695801.206615:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695801.206617:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695801.206620:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695801.206627:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695801.206629:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695801.206630:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695801.206634:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787695801.206636:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695801.206638:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695801.206644:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ac380. 00000020:00000040:2.0:1787695801.206646:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:2.0:1787695801.206651:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695801.206653:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695801.206655:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695801.206657:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695801.206662:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695801.206668:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661c00. 00000100:00000001:2.0:1787695801.206674:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695801.206682:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ac380 x1874531139524224/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695801.206691:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695801.206697:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695801.206702:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e778d47] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ac380 x1874531139524224/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695801.206726:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695801.206729:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695801.206732:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787695801.206734:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695801.206736:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695801.206738:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695801.206741:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3688 00000400:00000200:2.0:1787695801.206746:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695801.206749:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00001000:2.0:1787695801.206753:0:38:0:(import.c:2149:obd_at_measure()) add 20 to ffff8b1458081c18 time=117 v=15 (15 0 0 0) 00000100:00001000:2.0:1787695801.206757:0:38:0:(import.c:2215:obd_at_measure()) AT ffff8b1458081c18 change: old=15 new=20 delta=5 (val=20) hist 20 0 0 0 00000100:00000001:0.0:1787695801.206767:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1787695801.206769:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 15s 00000100:00000001:0.0:1787695801.206773:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695801.206776:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695801.206779:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:0.0:1787695801.206785:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ac380 x1874531139524224/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695801.206786:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695801.206793:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695801.206798:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695801.206799:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695801.206801:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695801.206807:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1787695801.206807:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695801.206811:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 02000000:00000001:0.0:1787695801.206812:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1787695801.206814:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1787695801.206815:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1787695801.206817:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000100:00100000:0.0:1787695801.206822:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139524224:192.168.201.140@tcp:38:kworker.0 00000020:00000001:2.0:1787695801.206822:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:2.0:1787695801.206828:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000001:0.0:1787695801.206829:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695801.206834:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:2.0:1787695801.206834:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695801.206836:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695801.206838:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695801.206839:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:0.0:1787695801.206844:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1787695801.206844:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 02000000:00000001:0.0:1787695801.206847:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695801.206867:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14475fbc00. 00000100:00080000:2.0:1787695801.206868:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000001:0.0:1787695801.206872:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695801.206874:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695801.206877:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1787695801.206880:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:2.0:1787695801.206880:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695801.206885:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695801.206886:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bb00. 00000100:00000001:2.0:1787695801.206889:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695801.206892:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000010:0.0:1787695801.206894:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f967f8. 00000008:00000001:2.0:1787695801.206895:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695801.206898:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695801.206901:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139524224, portal 10 00000100:00000010:2.0:1787695801.206903:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465af100. 00000100:00000001:0.0:1787695801.206905:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000040:2.0:1787695801.206907:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=25 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000100:00000001:0.0:1787695801.206908:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928801152 : -128555780750464 : ffff8b14465ac380) 02000000:00000001:2.0:1787695801.206912:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695801.206914:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695801.206916:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695801.206918:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:0.0:1787695801.206919:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ac380 x1874531139524224/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695801.206920:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695801.206927:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445660800. 00000100:00000001:0.0:1787695801.206929:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787695801.206932:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695801.206933:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695801.206938:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96220. 00000100:00100000:2.0:1787695801.206942:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465af100 x1874531139524352/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1787695801.206943:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139524224, offset 0 00000400:00000200:0.0:1787695801.206955:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695801.206955:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695801.206960:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695801.206964:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000094a5074e] to pc [ptlrpcd_rcv+-1] req@ffff8b14465af100 x1874531139524352/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695801.206967:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695801.206976:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695801.206981:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1787695801.206983:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695801.206986:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00080000:2.0:1787695801.206988:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1787695801.206992:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00080000:2.0:1787695801.206993:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787695801.206996:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1787695801.206997:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00080000:2.0:1787695801.207001:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1787695801.207005:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16905:16905:256:4294967295] 192.168.201.140@tcp LPNI seq info [16904:16904:8:4294967295] 00000100:00000001:2.0:1787695801.207006:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695801.207012:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3688 00000400:00000200:0.0:1787695801.207013:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695801.207025:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695801.207030:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695801.207034:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000800:00000010:0.0:1787695801.207036:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1445d7a400. 00000400:00000200:2.0:1787695801.207040:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695801.207043:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000800:00000200:0.0:1787695801.207048:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:2.0:1787695801.207050:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695801.207053:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.1:1787695801.207056:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:2.0:1787695801.207058:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695801.207061:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695801.207063:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695801.207064:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:2.0:1787695801.207065:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695801.207067:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:2.0:1787695801.207068:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:2.0:1787695801.207070:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:0.0:1787695801.207071:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695801.207073:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:0.0:1787695801.207076:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000100:00000001:2.0:1787695801.207076:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695801.207078:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695801.207082:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:0.0:1787695801.207085:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000040:2.0:1787695801.207088:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:2.0:1787695801.207092:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695801.207096:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298362821 00000100:00000001:2.0:1787695801.207098:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1787695801.207103:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8b1446635c60. 02000000:00000001:2.0:1787695801.207103:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695801.207109:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695801.207111:0:2402:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request now active for cpu 2 02000000:00000001:2.0:1787695801.207115:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695801.207118:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298362821 00000100:00000001:2.0:1787695801.207119:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695801.207125:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000010:0.0:1787695801.207126:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8b1446635360. 00000008:00000001:2.0:1787695801.207131:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695801.207134:0:2402:0:(niobuf.c:678:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request now active for cpu 3 00000100:00000001:2.0:1787695801.207136:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1787695801.207138:0:2390:0:(lib-socket.c:438:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.140:988 00000100:00000001:0.0:1787695801.207140:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:2.0:1787695801.207143:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465aea00. 00000100:00000001:0.0:1787695801.207147:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1787695801.207148:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:2.0:1787695801.207155:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787695801.207156:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695801.207161:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787695801.207162:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695801.207166:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695801.207167:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695801.207169:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695801.207170:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695801.207174:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1787695801.207174:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445660400. 00000100:00000001:2.0:1787695801.207179:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695801.207180:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695801.207184:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1787695801.207187:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465aea00 x1874531139524480/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695801.207188:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000400:00000200:3.0:1787695801.207195:0:2390:0:(lib-socket.c:438:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.140:988 00000100:00000001:2.0:1787695801.207198:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:0.0:1787695801.207199:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465af100 x1874531139524352/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695801.207202:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695801.207210:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a41ef0cb] to pc [ptlrpcd_rcv+-1] req@ffff8b14465aea00 x1874531139524480/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695801.207220:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695801.207224:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695801.207224:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695801.207230:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1787695801.207231:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1787695801.207234:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 02000000:00000001:0.0:1787695801.207235:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695801.207237:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:0.0:1787695801.207239:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695801.207240:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787695801.207245:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139524352:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695801.207246:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787695801.207251:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1787695801.207252:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3688 02000000:00000001:0.0:1787695801.207254:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695801.207257:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695801.207258:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:0.0:1787695801.207260:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:2.0:1787695801.207263:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 02000000:00000010:0.0:1787695801.207265:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14475f9c00. 00000100:00080000:2.0:1787695801.207266:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 02000000:00000001:0.0:1787695801.207270:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695801.207272:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695801.207276:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1787695801.207279:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:2.0:1787695801.207280:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695801.207284:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000010:0.0:1787695801.207286:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b580. 00000020:00000001:2.0:1787695801.207286:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695801.207288:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:2.0:1787695801.207292:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695801.207293:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000400:00000010:0.0:1787695801.207295:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96440. 00000020:00000001:2.0:1787695801.207295:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000200:0.0:1787695801.207300:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139524352, portal 10 00000020:00000001:2.0:1787695801.207302:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:2.0:1787695801.207305:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695801.207307:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695801.207310:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695801.207313:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695801.207313:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695801.207317:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928812800 : -128555780738816 : ffff8b14465af100) 00000020:00000040:2.0:1787695801.207318:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:2.0:1787695801.207322:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695801.207325:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695801.207326:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695801.207329:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1787695801.207331:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465af100 x1874531139524352/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695801.207331:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695801.207337:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695801.207340:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787695801.207346:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:2.0:1787695801.207347:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695801.207350:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695801.207351:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695801.207357:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 00000100:00000010:2.0:1787695801.207358:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465aed80. 00000100:00000200:0.0:1787695801.207361:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139524352, offset 0 00000020:00000040:2.0:1787695801.207362:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:2.0:1787695801.207388:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695801.207390:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695801.207391:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695801.207393:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695801.207396:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695801.207399:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 02000000:00000010:2.0:1787695801.207403:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663400. 00000400:00000200:0.0:1787695801.207407:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695801.207411:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695801.207418:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465aed80 x1874531139524608/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1787695801.207421:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695801.207428:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695801.207432:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:2.0:1787695801.207433:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1787695801.207436:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1787695801.207438:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695801.207440:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695801.207446:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16906:16906:255:4294967295] 192.168.201.140@tcp LPNI seq info [16905:16905:7:4294967295] 00000100:00000040:2.0:1787695801.207446:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bed91c58] to pc [ptlrpcd_rcv+-1] req@ffff8b14465aed80 x1874531139524608/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695801.207457:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695801.207466:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695801.207467:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695801.207473:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00080000:2.0:1787695801.207475:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:0.0:1787695801.207478:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1445d7a200. 00000100:00080000:2.0:1787695801.207479:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787695801.207483:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695801.207485:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695801.207486:0:38:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:0.0:1787695801.207490:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00080000:2.0:1787695801.207492:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3688 00000800:00000200:0.1:1787695801.207496:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000400:00000200:2.0:1787695801.207499:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695801.207503:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695801.207507:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787695801.207508:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695801.207510:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:0.0:1787695801.207514:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000100:00000001:2.0:1787695801.207514:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695801.207518:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695801.207522:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:0.0:1787695801.207526:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:2.0:1787695801.207526:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695801.207528:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:2.0:1787695801.207531:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787695801.207532:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298361670 00000020:00000040:2.0:1787695801.207533:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000020:00000001:2.0:1787695801.207538:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:0.0:1787695801.207539:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298362821 00000020:00000001:2.0:1787695801.207540:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:0.0:1787695801.207547:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298361670 00000100:00000040:2.0:1787695801.207548:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695801.207554:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298362821 00000100:00000001:2.0:1787695801.207554:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695801.207558:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695801.207560:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695801.207561:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695801.207563:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:2.0:1787695801.207564:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695801.207567:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695801.207569:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695801.207570:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787695801.207575:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695801.207575:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695801.207579:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787695801.207580:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695801.207583:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787695801.207584:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695801.207586:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695801.207588:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695801.207592:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:2.0:1787695801.207592:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787695801.207595:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00000001:2.0:1787695801.207597:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695801.207600:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695801.207601:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465aea00 x1874531139524480/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1787695801.207604:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ac000. 00000020:00000040:2.0:1787695801.207609:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000100:00000001:0.0:1787695801.207615:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1787695801.207615:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695801.207618:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695801.207619:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695801.207624:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695801.207625:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695801.207628:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787695801.207629:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695801.207633:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695801.207633:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695801.207640:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139524480:192.168.201.140@tcp:8:kworker.0 02000000:00000010:2.0:1787695801.207641:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445662800. 00000100:00000001:0.0:1787695801.207646:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787695801.207647:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695801.207650:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695801.207655:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695801.207655:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ac000 x1874531139524736/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787695801.207661:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787695801.207667:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000010:0.0:1787695801.207668:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14475f8800. 00000100:00000001:2.0:1787695801.207669:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695801.207674:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695801.207675:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a231b497] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ac000 x1874531139524736/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695801.207683:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695801.207686:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695801.207689:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bf78. 00000100:00080000:2.0:1787695801.207689:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695801.207695:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787695801.207697:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:0.0:1787695801.207698:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f962a8. 00000100:00080000:2.0:1787695801.207698:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695801.207704:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:0.0:1787695801.207705:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139524480, portal 4 00000100:00080000:2.0:1787695801.207708:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3688 00000100:00000001:0.0:1787695801.207709:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695801.207712:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928811008 : -128555780740608 : ffff8b14465aea00) 00000400:00000200:2.0:1787695801.207715:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:0.0:1787695801.207722:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465aea00 x1874531139524480/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787695801.207722:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00001000:2.0:1787695801.207725:0:38:0:(import.c:2149:obd_at_measure()) add 20 to ffff8b144515c418 time=117 v=15 (15 0 0 0) 00000100:00001000:2.0:1787695801.207734:0:38:0:(import.c:2215:obd_at_measure()) AT ffff8b144515c418 change: old=15 new=20 delta=5 (val=20) hist 20 0 0 0 00000100:00000001:0.0:1787695801.207737:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695801.207741:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00080000:2.0:1787695801.207742:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 15s 00000400:00000010:0.0:1787695801.207744:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96660. 00000100:00000200:0.0:1787695801.207749:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139524480, offset 0 00000100:00000040:2.0:1787695801.207752:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695801.207761:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695801.207766:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1787695801.207768:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:2.0:1787695801.207773:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695801.207774:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695801.207779:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1787695801.207780:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:0.0:1787695801.207783:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695801.207788:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000020:00000001:2.0:1787695801.207788:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695801.207792:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000400:00000200:0.0:1787695801.207793:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695801.207798:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16907:16907:254:4294967295] 192.168.201.140@tcp LPNI seq info [16906:16906:6:4294967295] 00000020:00000001:2.0:1787695801.207799:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:0.0:1787695801.207803:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000040:2.0:1787695801.207804:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:2.0:1787695801.207809:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000400:00000200:0.0:1787695801.207812:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000020:00000001:2.0:1787695801.207815:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000800:00000200:0.0:1787695801.207819:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695801.207823:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1445d7aa00. 00000100:00000040:2.0:1787695801.207824:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000800:00000200:0.0:1787695801.207830:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:2.0:1787695801.207832:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.1:1787695801.207836:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695801.207839:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695801.207840:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000040:0.0:1787695801.207842:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000100:00000001:2.0:1787695801.207847:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:0.0:1787695801.207848:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000040:2.0:1787695801.207862:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00000001:0.0:1787695801.207865:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298361700 00000100:00000001:0.0:1787695801.207869:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298362821 00000100:00080000:2.0:1787695801.207869:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695801.207875:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695801.207876:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298361700 00000100:00000001:0.0:1787695801.207881:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298362821 02000000:00000001:2.0:1787695801.207881:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787695801.207885:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695801.207886:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695801.207887:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695801.207893:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787695801.207893:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695801.207897:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000040:0.0:1787695801.207900:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465aed80 x1874531139524608/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787695801.207901:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787695801.207907:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695801.207911:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695801.207915:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695801.207917:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695801.207917:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465adf80. 02000000:00000001:0.0:1787695801.207919:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695801.207923:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1787695801.207923:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=26 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:0.0:1787695801.207927:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695801.207928:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695801.207930:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695801.207931:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695801.207932:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695801.207934:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695801.207938:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139524608:192.168.201.140@tcp:8:kworker.0 02000000:00000010:2.0:1787695801.207941:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445662000. 00000100:00000001:0.0:1787695801.207942:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695801.207945:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695801.207947:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695801.207947:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695801.207950:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695801.207953:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14475f9800. 02000000:00000001:0.0:1787695801.207956:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695801.207960:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00100000:2.0:1787695801.207964:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465adf80 x1874531139524864/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:0.0:1787695801.207965:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b738. 00000400:00000010:0.0:1787695801.207970:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96c38. 00000100:00000200:0.0:1787695801.207975:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139524608, portal 4 00000100:00000001:2.0:1787695801.207976:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695801.207977:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695801.207980:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695801.207981:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928811904 : -128555780739712 : ffff8b14465aed80) 00000100:00000040:2.0:1787695801.207988:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ee5f8f2] to pc [ptlrpcd_rcv+-1] req@ffff8b14465adf80 x1874531139524864/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695801.207991:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465aed80 x1874531139524608/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695801.208010:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695801.208012:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695801.208014:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1787695801.208018:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000040:2.0:1787695801.208020:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000010:0.0:1787695801.208022:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96990. 00000100:00000200:0.0:1787695801.208025:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139524608, offset 0 00000100:00000040:2.0:1787695801.208025:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1787695801.208031:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:2.0:1787695801.208032:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:0.0:1787695801.208036:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:2.0:1787695801.208037:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3699) 00000400:00000200:0.0:1787695801.208041:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695801.208044:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695801.208046:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695801.208048:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695801.208051:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695801.208054:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16908:16908:253:4294967295] 192.168.201.140@tcp LPNI seq info [16907:16907:5:4294967295] 00000400:00000200:0.0:1787695801.208056:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695801.208061:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695801.208063:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695801.208066:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1445d7a100. 00000800:00000200:0.0:1787695801.208073:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695801.208076:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695801.208078:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695801.208080:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:0.0:1787695801.208082:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695801.208083:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298361372 00000100:00000001:0.0:1787695801.208085:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298362821 00000100:00000001:0.0:1787695801.208087:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298361372 00000100:00000001:0.0:1787695801.208088:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298362821 00000100:00000001:0.0:1787695801.208090:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695801.208091:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695801.208093:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695801.208094:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695801.208096:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695801.208097:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695801.208099:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695801.208100:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695801.208101:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695801.208103:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ac000 x1874531139524736/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695801.208108:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695801.208109:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695801.208111:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695801.208112:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695801.208113:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695801.208115:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139524736:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695801.208117:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695801.208118:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695801.208120:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695801.208121:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695801.208125:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695801.208127:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695801.208130:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695801.208137:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b058. 00000400:00000010:0.0:1787695801.208141:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695801.208144:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139524736, portal 10 00000100:00000001:0.0:1787695801.208146:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695801.208148:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928800256 : -128555780751360 : ffff8b14465ac000) 00000100:00000040:0.0:1787695801.208155:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ac000 x1874531139524736/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695801.208162:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695801.208165:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695801.208167:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96000. 00000100:00000200:0.0:1787695801.208170:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139524736, offset 0 00000400:00000200:0.0:1787695801.208175:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695801.208179:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695801.208182:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695801.208186:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695801.208188:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695801.208191:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695801.208193:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695801.208196:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16909:16909:252:4294967295] 192.168.201.140@tcp LPNI seq info [16908:16908:4:4294967295] 00000400:00000200:0.0:1787695801.208200:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695801.208207:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695801.208212:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695801.208216:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1445d7af00. 00000800:00000200:0.0:1787695801.208220:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695801.208224:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695801.208227:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695801.208229:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:0.0:1787695801.208231:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695801.208233:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298361558 00000100:00000001:0.0:1787695801.208235:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298362821 00000100:00000001:0.0:1787695801.208236:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298361558 00000100:00000001:0.0:1787695801.208243:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298362821 00000100:00000001:0.0:1787695801.208245:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695801.208246:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695801.208248:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695801.208251:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465adf80 x1874531139524864/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695801.208256:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695801.208257:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695801.208259:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695801.208260:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695801.208261:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695801.208263:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139524864:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695801.208265:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695801.208267:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695801.208268:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695801.208269:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695801.208272:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695801.208274:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695801.208277:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695801.208280:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617be18. 00000400:00000010:0.0:1787695801.208284:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96bb0. 00000100:00000200:0.0:1787695801.208287:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139524864, portal 10 00000100:00000001:0.0:1787695801.208289:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695801.208290:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928808320 : -128555780743296 : ffff8b14465adf80) 00000100:00000040:0.0:1787695801.208296:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465adf80 x1874531139524864/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695801.208303:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695801.208306:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695801.208308:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f963b8. 00000100:00000200:0.0:1787695801.208311:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139524864, offset 0 00000400:00000200:0.0:1787695801.208316:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695801.208320:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695801.208329:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695801.208331:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695801.208333:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695801.208335:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695801.208337:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695801.208339:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16910:16910:251:4294967295] 192.168.201.140@tcp LPNI seq info [16909:16909:3:4294967295] 00000400:00000200:0.0:1787695801.208342:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695801.208347:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695801.208350:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695801.208353:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1444fe2100. 00000800:00000200:0.0:1787695801.208355:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695801.208358:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695801.208360:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695801.208377:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:0.0:1787695801.208381:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695801.208383:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298361537 00000100:00000001:0.0:1787695801.208385:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298362821 00000100:00000001:0.0:1787695801.208387:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298361537 00000100:00000001:0.0:1787695801.208389:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298362821 00000100:00000001:0.0:1787695801.208390:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695801.208392:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695801.208394:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695801.208395:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695801.208398:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695801.208399:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695801.208400:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695801.208401:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695801.208402:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695801.208404:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695801.270426:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695801.270433:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695801.270435:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695801.270439:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695801.270441:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695801.270444:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695801.270446:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695801.270448:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695801.270450:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695801.910468:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695801.910474:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695801.910478:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695801.910479:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695801.910482:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695801.910483:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695801.910485:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695801.910486:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695801.910489:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695801.910491:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695801.910493:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695801.910498:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695801.910501:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695801.910502:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695801.910505:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695801.910508:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695801.910510:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695801.910512:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695801.910535:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695801.910537:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695801.910600:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695801.910606:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695801.910609:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787695802.230453:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695802.230471:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695802.230473:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695802.230478:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695802.230491:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695802.230497:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695802.230499:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695802.294547:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695802.294555:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695802.294557:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695802.294562:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695802.294565:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695802.294569:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695802.294572:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695802.294574:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695802.294576:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695802.742494:0:41781:0:(connection.c:75:cpu_latency_work()) work item of ffff8b1460486330 (cpu 3) has reached its deadline 4298362821, at 4298362880 00000100:00000010:3.0:1787695802.742509:0:41781:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8b1446635360. 00000100:00000040:2.0:1787695802.742616:0:41120:0:(connection.c:75:cpu_latency_work()) work item of ffff8b1460486220 (cpu 2) has reached its deadline 4298362821, at 4298362880 00000100:00000010:2.0:1787695802.742624:0:41120:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff8b1446635c60. 00010000:00000001:1.0:1787695802.934533:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695802.934541:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695802.934547:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695802.934551:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695802.934553:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695802.934557:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695802.934560:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695802.934562:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695802.934564:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695802.934566:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695802.934569:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695802.934570:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695802.934573:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695802.934574:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695802.934577:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695802.934578:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695802.934581:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695802.934581:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695802.934683:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695802.934691:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695802.934696:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787695803.255427:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695803.255447:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695803.255450:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695803.255457:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695803.255460:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695803.255467:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695803.255470:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695803.318451:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695803.318462:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695803.318465:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695803.318473:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695803.318477:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695803.318482:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695803.318486:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695803.318489:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695803.318493:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695803.958452:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695803.958462:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695803.958465:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695803.958468:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695803.958474:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695803.958476:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695803.958478:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695803.958480:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695803.958484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695803.958485:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695803.958488:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695803.958489:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695803.958492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695803.958494:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695803.958497:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695803.958498:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695803.958500:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695803.958501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695803.958582:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695803.958588:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695803.958591:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787695804.086744:0:2390:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.201.140:988 00000400:00000100:3.0:1787695804.086823:0:2390:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.140@tcp at host 192.168.201.140 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:3.0:1787695804.086832:0:2390:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695804.086836:0:2390:0:(router.c:1791:lnet_notify()) 192.168.201.40@tcp notifying 192.168.201.140@tcp: down 00000400:00000200:3.0:1787695804.086840:0:2390:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695804.086844:0:2390:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:3.0:1787695804.086847:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:3.0:1787695804.086851:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1445d7a400. 00000400:00000200:3.0:1787695804.086856:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1787695804.086860:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1787695804.086867:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96220 00000400:00000010:3.0:1787695804.086870:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96220. 00000100:00000001:3.0:1787695804.086875:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:3.0:1787695804.086884:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14465ac380 x1874531139524224/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695804.086918:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695804.086923:0:2390:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ac380 x1874531139524224/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695826 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695804.086932:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695804.086934:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787695804.086943:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:3.0:1787695804.086947:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1445d7a200. 00000400:00000200:3.0:1787695804.086951:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1787695804.086955:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1787695804.086959:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:3.0:1787695804.086961:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:3.0:1787695804.086964:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787695804.086970:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:3.0:1787695804.086970:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14465af100 x1874531139524352/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.086977:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695804.086984:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695804.086989:0:2390:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465af100 x1874531139524352/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695826 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.087000:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787695804.087013:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695804.087014:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695801/real 1787695804] req@ffff8b14465af100 x1874531139524352/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695804.087018:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787695804.087030:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000400:00000001:0.0:1787695804.087034:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000010:3.0:1787695804.087034:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1445d7aa00. 00000400:00000001:0.0:1787695804.087039:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:3.0:1787695804.087041:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000010:0.0:1787695804.087046:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b580. 00000400:00000200:3.0:1787695804.087048:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1787695804.087054:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96660 00000400:00000200:0.0:1787695804.087055:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96440 00000400:00000010:3.0:1787695804.087057:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96660. 00000400:00000010:0.0:1787695804.087060:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96440. 00000100:00000001:3.0:1787695804.087064:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787695804.087066:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787695804.087073:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14465aea00 x1874531139524480/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695804.087075:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465af100 x1874531139524352/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695804.087099:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000200:0.0:1787695804.087105:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465af100 x1874531139524352/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695804.087109:0:2390:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465aea00 x1874531139524480/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695826 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.087123:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787695804.087126:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695804.087128:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695804.087130:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695804.087134:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695804.087137:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000800:00000100:3.0:1787695804.087139:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:3.0:1787695804.087143:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1445d7a100. 00000400:00000200:3.0:1787695804.087146:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1787695804.087154:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000100:00100000:0.0:1787695804.087158:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465af100 x1874531139524352/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695804.087159:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96990 00000400:00000010:3.0:1787695804.087164:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96990. 00000100:00000001:3.0:1787695804.087169:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787695804.087175:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1787695804.087178:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14465aed80 x1874531139524608/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695804.087186:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465af100 x1874531139524352/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695804.087192:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787695804.087200:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:3.0:1787695804.087200:0:2390:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465aed80 x1874531139524608/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695826 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.087208:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.087211:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695804.087212:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.087215:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695804.087217:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695804.087222:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695804.087228:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695804.087234:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787695804.087236:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695804.087240:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1787695804.087243:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465af100 x1874531139524352/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695804.087256:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139524352:192.168.201.140@tcp:38:kworker.0 00000800:00000100:3.0:1787695804.087262:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000100:00000001:0.0:1787695804.087265:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000800:00000010:3.0:1787695804.087267:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1445d7af00. 00000400:00000200:3.0:1787695804.087273:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00100000:0.0:1787695804.087274:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695801/real 1787695804] req@ffff8b14465ac380 x1874531139524224/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695804.087282:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1787695804.087291:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96000 00000400:00000010:3.0:1787695804.087294:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96000. 00000400:00000001:0.0:1787695804.087295:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787695804.087297:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000400:00000001:0.0:1787695804.087298:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695804.087303:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bb00. 00000100:00000200:3.0:1787695804.087306:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14465ac000 x1874531139524736/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695804.087308:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f967f8 00000400:00000010:0.0:1787695804.087313:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f967f8. 00000100:00000001:0.0:1787695804.087318:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787695804.087323:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000200:0.0:1787695804.087327:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ac380 x1874531139524224/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695804.087332:0:2390:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ac000 x1874531139524736/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695826 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695804.087346:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695804.087348:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000200:0.0:1787695804.087401:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ac380 x1874531139524224/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.087411:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695804.087413:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.087414:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695804.087416:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695804.087421:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ac380 x1874531139524224/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.087428:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695804.087433:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ac380 x1874531139524224/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.087440:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695804.087443:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.087444:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695804.087446:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.087448:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695804.087450:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695804.087454:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695804.087458:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695804.087462:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695804.087467:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ac380 x1874531139524224/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695804.087477:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139524224:192.168.201.140@tcp:38:kworker.0 00000800:00000100:3.0:1787695804.087482:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000100:00000001:0.0:1787695804.087486:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1787695804.087488:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1444fe2100. 00000100:00000001:0.0:1787695804.087492:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695804.087501:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465af100 x1874531139524352/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695804.087504:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:3.0:1787695804.087514:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:3.0:1787695804.087520:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f963b8 00000400:00000010:3.0:1787695804.087525:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f963b8. 00000100:00000001:3.0:1787695804.087533:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787695804.087537:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695804.087543:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000200:3.0:1787695804.087543:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14465adf80 x1874531139524864/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695826 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1787695804.087550:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14475f9c00. 02000000:00000001:0.0:1787695804.087559:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1787695804.087560:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1787695804.087565:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000040:3.0:1787695804.087566:0:2390:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465adf80 x1874531139524864/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695826 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1787695804.087569:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=24 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695804.087575:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695804.087579:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445660800. 00000100:00000001:3.0:1787695804.087581:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695804.087584:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1787695804.087586:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1787695804.087588:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695804.087592:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465af100. 00000100:00000001:0.0:1787695804.087598:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695804.087599:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.087601:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695804.087607:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ac380 x1874531139524224/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.087614:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695804.087615:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695804.087617:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14475fbc00. 02000000:00000001:0.0:1787695804.087620:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695804.087621:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695804.087622:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695804.087625:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695804.087626:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661c00. 02000000:00000001:0.0:1787695804.087628:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695804.087630:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695804.087631:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ac380. 00000100:00000001:0.0:1787695804.087634:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695804.087635:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.087638:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695804.087640:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695804.087641:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695804.087643:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695804.087647:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695801/real 1787695804] req@ffff8b14465ac000 x1874531139524736/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695804.087666:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695804.087667:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695804.087669:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b058. 00000400:00000200:0.0:1787695804.087672:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695804.087673:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695804.087676:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695804.087679:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ac000 x1874531139524736/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695804.087688:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ac000 x1874531139524736/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.087695:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695804.087697:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.087698:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695804.087700:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695804.087703:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ac000 x1874531139524736/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.087709:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695804.087714:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ac000 x1874531139524736/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.087721:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695804.087723:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.087725:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695804.087726:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.087728:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695804.087730:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695804.087733:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695804.087737:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695804.087739:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695804.087750:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ac000 x1874531139524736/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695804.087758:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139524736:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695804.087763:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695804.087766:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695801/real 1787695804] req@ffff8b14465adf80 x1874531139524864/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695804.087776:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695804.087777:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695804.087779:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617be18. 00000400:00000200:0.0:1787695804.087783:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96bb0 00000400:00000010:0.0:1787695804.087785:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96bb0. 00000100:00000001:0.0:1787695804.087787:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695804.087791:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465adf80 x1874531139524864/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695804.087813:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465adf80 x1874531139524864/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.087821:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695804.087823:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.087824:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695804.087826:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695804.087830:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465adf80 x1874531139524864/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.087837:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695804.087842:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465adf80 x1874531139524864/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.087849:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695804.087852:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.087861:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695804.087862:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.087864:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695804.087866:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695804.087869:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695804.087874:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695804.087876:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695804.087881:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465adf80 x1874531139524864/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695804.087889:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139524864:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695804.087894:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695804.087898:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695801/real 1787695804] req@ffff8b14465aea00 x1874531139524480/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695804.087908:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695804.087909:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695804.087912:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bf78. 00000400:00000200:0.0:1787695804.087915:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f962a8 00000400:00000010:0.0:1787695804.087916:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f962a8. 00000100:00000001:0.0:1787695804.087919:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695804.087922:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465aea00 x1874531139524480/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695804.087932:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465aea00 x1874531139524480/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.087939:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695804.087940:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.087942:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695804.087943:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695804.087948:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465aea00 x1874531139524480/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.087967:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695804.087972:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465aea00 x1874531139524480/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.087979:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695804.087981:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.087983:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695804.087984:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.087986:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695804.087988:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695804.087991:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695804.087996:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695804.087998:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695804.088002:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465aea00 x1874531139524480/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695804.088011:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139524480:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695804.088016:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695804.088019:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695801/real 1787695804] req@ffff8b14465aed80 x1874531139524608/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695804.088027:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695804.088028:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695804.088030:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b738. 00000400:00000200:0.0:1787695804.088033:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96c38 00000400:00000010:0.0:1787695804.088034:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96c38. 00000100:00000001:0.0:1787695804.088037:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695804.088040:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465aed80 x1874531139524608/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695804.088059:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465aed80 x1874531139524608/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.088065:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695804.088066:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.088068:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695804.088069:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695804.088073:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465aed80 x1874531139524608/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.088079:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695804.088083:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465aed80 x1874531139524608/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.088090:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695804.088092:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.088094:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695804.088095:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.088097:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695804.088099:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695804.088101:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695804.088104:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695804.088106:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695804.088111:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465aed80 x1874531139524608/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695804.088119:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139524608:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695804.088124:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695804.088126:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695804.088129:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ac000 x1874531139524736/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.088147:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695804.088148:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695804.088150:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695804.088154:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695804.088164:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695804.088165:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695804.088167:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695804.088169:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445662800. 02000000:00000001:0.0:1787695804.088172:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695804.088173:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695804.088175:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ac000. 00000100:00000001:0.0:1787695804.088178:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695804.088180:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.088182:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695804.088186:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465adf80 x1874531139524864/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.088194:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695804.088195:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695804.088197:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695804.088200:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695804.088201:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695804.088203:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=25 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695804.088205:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695804.088206:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445662000. 02000000:00000001:0.0:1787695804.088208:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695804.088210:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695804.088211:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465adf80. 00000100:00000001:0.0:1787695804.088214:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695804.088215:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.088217:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695804.088219:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465aea00 x1874531139524480/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.088224:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695804.088225:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695804.088232:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14475f8800. 02000000:00000001:0.0:1787695804.088237:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695804.088238:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695804.088240:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:0.0:1787695804.088242:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695804.088244:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445660400. 02000000:00000001:0.0:1787695804.088246:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695804.088247:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695804.088249:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465aea00. 00000100:00000001:0.0:1787695804.088251:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695804.088253:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.088255:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695804.088260:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465aed80 x1874531139524608/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695826 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695804.088266:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695804.088267:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695804.088269:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14475f9800. 02000000:00000001:0.0:1787695804.088271:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695804.088272:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695804.088274:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:0.0:1787695804.088275:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695804.088276:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663400. 02000000:00000001:0.0:1787695804.088279:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695804.088280:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695804.088281:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465aed80. 00000100:00000001:0.0:1787695804.088283:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695804.088284:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.088286:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695804.088288:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695804.088290:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695804.088291:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695804.088293:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695804.262459:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695804.262472:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695804.262474:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695804.262480:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695804.262482:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695804.262502:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695804.262506:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695804.342492:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695804.342500:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695804.342503:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695804.342509:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695804.342529:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695804.342532:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695804.342535:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695804.342538:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695804.342541:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695804.982474:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695804.982482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695804.982487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695804.982488:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695804.982491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695804.982492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695804.982495:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695804.982496:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695804.982498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695804.982499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695804.982502:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695804.982503:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695804.982557:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695804.982562:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695804.982565:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787695805.302424:0:2402:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695805.302429:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695805.302441:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695780/real 1787695780] req@ffff8b1446664380 x1874531139520768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695805 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695805.302456:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695805.302458:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695805.302460:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bf20. 00000400:00000200:0.0:1787695805.302465:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96d48 00000400:00000010:0.0:1787695805.302467:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96d48. 00000100:00000001:0.0:1787695805.302470:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695805.302473:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446664380 x1874531139520768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695805 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695805.302480:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446664380 x1874531139520768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695805 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.302485:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695805.302486:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.302488:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695805.302490:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695805.302492:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446664380 x1874531139520768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695805 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.302497:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.302504:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695805.302505:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.302507:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695805.302517:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695805.302519:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695805.302524:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446664380 x1874531139520768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695805 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.302530:0:2402:0:(client.c:2142:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695805.302532:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.302541:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695805.302543:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.302544:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695805.302547:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695805.302550:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695805.302565:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695805.302567:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695805.302571:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446664380 x1874531139520768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695805 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695805.302579:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446664380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139520768:192.168.201.140@tcp:250:kworker.0 00000100:00000001:0.0:1787695805.302586:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.302589:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695805.302595:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446664380 x1874531139520768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695805 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695805.302595:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695805.302602:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787695805.302605:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695805.302609:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695805.302611:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1787695805.302612:0:38:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:0.0:1787695805.302615:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:3.0:1787695805.302619:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3673 02000000:00000010:0.0:1787695805.302622:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14475fa400. 02000000:00000001:0.0:1787695805.302629:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:3.0:1787695805.302629:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:0.0:1787695805.302632:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695805.302636:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:0.0:1787695805.302640:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000400:00000200:3.0:1787695805.302640:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 02000000:00000010:0.0:1787695805.302645:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446544400. 00000100:00080000:3.0:1787695805.302646:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 02000000:00000001:0.0:1787695805.302652:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:3.0:1787695805.302653:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695805.302657:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787695805.302659:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000010:0.0:1787695805.302661:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446664380. 00000100:00000040:3.0:1787695805.302665:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695805.302668:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787695805.302670:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695805.302672:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787695805.302675:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787695805.302676:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1787695805.302677:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000100:00000001:0.0:1787695805.302680:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:3.0:1787695805.302681:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787695805.302684:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695805.302685:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000100:00000001:0.0:1787695805.302687:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787695805.302689:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000001:0.0:1787695805.302690:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695805.302693:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787695805.302697:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695805.302699:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695805.302701:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695805.302702:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695805.302705:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787695805.302713:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695805.302715:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695805.302717:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695805.302720:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695805.302722:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695805.302724:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787695805.302726:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695805.302729:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4000. 00000020:00000040:3.0:1787695805.302732:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787695805.302742:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695805.302743:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695805.302748:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695805.302750:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695805.302753:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695805.302759:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f7400. 00000100:00000001:3.0:1787695805.302764:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695805.302769:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4000 x1874531139524992/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695805.302777:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695805.302795:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695805.302799:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5a8f356] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4000 x1874531139524992/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695805.302812:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695805.302815:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695805.302818:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695805.302819:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695805.302820:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695805.302822:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695805.302824:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3694 00000400:00000200:3.0:1787695805.302828:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695805.302831:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695805.302833:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787695805.302835:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1787695805.302841:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695805.302844:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695805.302844:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695805.302847:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1787695805.302848:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695805.302851:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000040:0.0:1787695805.302854:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4000 x1874531139524992/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695805.302855:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695805.302857:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:3.0:1787695805.302860:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695805.302863:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:3.0:1787695805.302867:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000001:0.0:1787695805.302870:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1787695805.302870:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000001:0.0:1787695805.302872:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695805.302876:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695805.302877:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787695805.302879:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787695805.302881:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1787695805.302882:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695805.302885:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695805.302886:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695805.302891:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00100000:0.0:1787695805.302892:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139524992:192.168.201.140@tcp:250:kworker.0 00000100:00080000:3.0:1787695805.302898:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695805.302899:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695805.302902:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695805.302904:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695805.302906:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787695805.302908:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695805.302911:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450775c00. 02000000:00000001:3.0:1787695805.302913:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787695805.302914:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695805.302920:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1787695805.302921:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787695805.302925:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695805.302926:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bf20. 00000100:00000001:3.0:1787695805.302928:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:0.0:1787695805.302932:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96d48. 00000008:00100000:3.0:1787695805.302933:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695805.302938:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695805.302939:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695805.302940:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139524992, portal 25 00000100:00000010:3.0:1787695805.302943:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4a80. 00000100:00000001:0.0:1787695805.302944:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695805.302946:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930143744 : -128555779407872 : ffff8b14466f4000) 00000020:00000040:3.0:1787695805.302948:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695805.302951:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000040:0.0:1787695805.302953:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4000 x1874531139524992/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695805.302953:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695805.302957:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695805.302959:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695805.302963:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.302966:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000010:3.0:1787695805.302968:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f5800. 00000100:00000040:0.0:1787695805.302970:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695805.302973:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96c38. 00000100:00000200:0.0:1787695805.302976:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139524992, offset 0 00000100:00000001:3.0:1787695805.302980:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695805.302984:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00100000:3.0:1787695805.302989:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4a80 x1874531139525120/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1787695805.302994:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695805.303001:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1787695805.303002:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:3.0:1787695805.303005:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695805.303010:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1787695805.303011:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ad0d67] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4a80 x1874531139525120/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695805.303015:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:3.0:1787695805.303020:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695805.303022:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695805.303026:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:0.0:1787695805.303027:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1787695805.303028:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1787695805.303031:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00080000:3.0:1787695805.303031:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695805.303037:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:0.0:1787695805.303040:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16911:16911:256:4294967295] 192.168.201.140@tcp LPNI seq info [16910:16910:8:4294967295] 00000100:00080000:3.0:1787695805.303042:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3694 00000400:00000200:0.0:1787695805.303049:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695805.303050:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695805.303054:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695805.303056:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695805.303060:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695805.303065:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1787695805.303067:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000040:3.0:1787695805.303070:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695805.303074:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000200:0.0:1787695805.303078:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000020:00000001:3.0:1787695805.303079:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000010:0.0:1787695805.303083:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1444fe2500. 00000020:00000040:3.0:1787695805.303084:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:3.0:1787695805.303088:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695805.303091:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000800:00000200:0.0:1787695805.303092:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000020:00000001:3.0:1787695805.303097:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000800:00000200:0.0:1787695805.303098:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 6 secs later) 00000020:00000001:3.0:1787695805.303104:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000800:00000200:0.1:1787695805.303107:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695805.303111:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 6 secs later) 00000100:00000040:3.0:1787695805.303113:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000800:00000100:0.0:1787695805.303118:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695805.303119:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:0.0:1787695805.303121:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1444fe2500. 00000400:00000200:0.0:1787695805.303124:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695805.303128:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695805.303132:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96c38 00000100:00000040:3.0:1787695805.303133:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000010:0.0:1787695805.303134:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96c38. 00000100:00000001:0.0:1787695805.303140:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787695805.303140:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695805.303144:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00000200:0.0:1787695805.303147:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4000 x1874531139524992/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695805.303150:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695805.303155:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695805.303159:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787695805.303163:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1787695805.303163:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1787695805.303168:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4000 x1874531139524992/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695830 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695805.303168:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.303177:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.303178:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787695805.303180:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787695805.303184:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787695805.303184:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1787695805.303190:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000008:00000001:3.0:1787695805.303190:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695805.303195:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.303196:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298377016 00000100:00000010:3.0:1787695805.303199:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7100. 00000100:00000010:0.0:1787695805.303201:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8b1446635c60. 00000020:00000040:3.0:1787695805.303203:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=25 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695805.303207:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695805.303208:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695805.303209:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787695805.303210:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request now active for cpu 2 02000000:00000001:3.0:1787695805.303211:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695805.303215:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.303217:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298377016 02000000:00000010:3.0:1787695805.303220:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f4400. 00000100:00000010:0.0:1787695805.303222:0:2402:0:(niobuf.c:666:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff8b1446635360. 00000100:00000001:3.0:1787695805.303224:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695805.303230:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7100 x1874531139525248/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695805.303231:0:2402:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request now active for cpu 3 00000100:00000001:0.0:1787695805.303235:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695805.303240:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.303245:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787695805.303246:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695805.303249:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695805.303252:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695805/real 1787695805] req@ffff8b14466f4000 x1874531139524992/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695805.303256:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000004e8abd7] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7100 x1874531139525248/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695805.303268:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695805.303271:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787695805.303272:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695805.303274:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bf20. 00000100:00080000:3.0:1787695805.303275:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1787695805.303277:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96d48 00000100:00080000:3.0:1787695805.303279:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000010:0.0:1787695805.303280:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96d48. 00000100:00000001:3.0:1787695805.303282:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787695805.303283:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695805.303287:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4000 x1874531139524992/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695805.303291:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695805.303297:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:0.0:1787695805.303300:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4000 x1874531139524992/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695805.303301:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3694 00000400:00000200:3.0:1787695805.303311:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695805.303315:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:3.0:1787695805.303316:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:0.0:1787695805.303319:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695805.303320:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695805.303325:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695805.303326:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695805.303328:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695805.303329:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695805.303331:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695805.303334:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695805.303337:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695805.303338:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000100:00100000:0.0:1787695805.303339:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4000 x1874531139524992/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695805.303340:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695805.303343:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:3.0:1787695805.303347:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:3.0:1787695805.303351:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:3.0:1787695805.303357:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695805.303361:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695805.303361:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695805.303388:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000040:0.0:1787695805.303390:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4000 x1874531139524992/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695805.303392:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695805.303397:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:3.0:1787695805.303403:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695805.303405:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787695805.303407:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.303408:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787695805.303409:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787695805.303412:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695805.303414:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787695805.303414:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695805.303418:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:3.0:1787695805.303419:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695805.303421:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695805.303423:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787695805.303425:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:3.0:1787695805.303426:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695805.303431:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787695805.303431:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695805.303437:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695805.303442:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000010:3.0:1787695805.303443:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6680. 00000100:00000040:0.0:1787695805.303446:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4000 x1874531139524992/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1787695805.303447:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:3.0:1787695805.303452:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695805.303455:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695805.303458:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:0.0:1787695805.303461:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139524992:192.168.201.140@tcp:250:kworker.0 02000000:00000001:3.0:1787695805.303461:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695805.303464:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.303468:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1787695805.303469:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f5000. 00000100:00000001:0.0:1787695805.303470:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787695805.303475:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695805.303476:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4000 x1874531139524992/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695805.303484:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6680 x1874531139525376/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695805.303485:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695805.303488:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695805.303490:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695805.303495:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1787695805.303496:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000020:00000001:0.0:1787695805.303498:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:3.0:1787695805.303500:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1787695805.303501:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:0.0:1787695805.303505:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000040:3.0:1787695805.303506:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f6d2abf9] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6680 x1874531139525376/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1787695805.303507:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f7400. 02000000:00000001:0.0:1787695805.303511:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695805.303513:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695805.303516:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4000. 00000100:00000001:0.0:1787695805.303520:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787695805.303521:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.303523:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695805.303525:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695805.303526:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695805.303531:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1787695805.303532:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695805.303533:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787695805.303535:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695805.303536:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695805.303539:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695805.303540:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787695805.303541:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1787695805.303544:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3694 00000100:00000001:0.0:1787695805.303545:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695805.303551:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4a80 x1874531139525120/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695805.303551:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695805.303556:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695805.303561:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695805.303565:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695805.303566:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695805.303568:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695805.303571:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695805.303575:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787695805.303581:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1787695805.303584:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000100:00000001:0.0:1787695805.303585:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695805.303587:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1787695805.303591:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:3.0:1787695805.303591:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:3.0:1787695805.303593:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 02000000:00000001:0.0:1787695805.303596:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1787695805.303596:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:3.0:1787695805.303602:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695805.303603:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695805.303604:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695805.303611:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00100000:0.0:1787695805.303612:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139525120:192.168.201.140@tcp:38:kworker.0 00000100:00000001:3.0:1787695805.303612:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695805.303617:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00000001:0.0:1787695805.303622:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1787695805.303623:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000001:0.0:1787695805.303626:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787695805.303628:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695805.303629:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695805.303633:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787695805.303633:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:0.0:1787695805.303636:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695805.303639:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695805.303639:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695805.303646:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695805.303647:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695805.303651:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bf20. 00000100:00000001:3.0:1787695805.303651:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:0.0:1787695805.303656:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96d48. 00000008:00100000:3.0:1787695805.303656:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000200:0.0:1787695805.303660:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139525120, portal 10 00000008:00000001:3.0:1787695805.303662:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.303663:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695805.303665:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930146432 : -128555779405184 : ffff8b14466f4a80) 00000100:00000001:3.0:1787695805.303667:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695805.303672:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5880. 00000100:00000040:0.0:1787695805.303673:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4a80 x1874531139525120/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1787695805.303675:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:3.0:1787695805.303680:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695805.303682:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.303685:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787695805.303685:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695805.303688:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:0.0:1787695805.303689:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695805.303691:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695805.303693:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96c38. 00000100:00000200:0.0:1787695805.303696:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139525120, offset 0 02000000:00000010:3.0:1787695805.303697:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f5400. 00000400:00000200:0.0:1787695805.303702:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695805.303702:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695805.303707:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00100000:3.0:1787695805.303710:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5880 x1874531139525504/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1787695805.303714:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695805.303720:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695805.303724:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:3.0:1787695805.303724:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695805.303728:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695805.303729:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695805.303731:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000040:3.0:1787695805.303738:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000827e7dbe] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5880 x1874531139525504/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695805.303746:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16912:16912:256:4294967295] 192.168.201.140@tcp LPNI seq info [16911:16911:8:4294967295] 00000100:00000001:3.0:1787695805.303751:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695805.303755:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1787695805.303756:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:3.0:1787695805.303759:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695805.303762:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695805.303764:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695805.303769:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:0.0:1787695805.303773:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00080000:3.0:1787695805.303774:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3694 00000800:00000200:0.0:1787695805.303799:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000400:00000200:3.0:1787695805.303801:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000800:00000010:0.0:1787695805.303804:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1444fe2500. 00000400:00000200:3.0:1787695805.303806:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695805.303810:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000800:00000200:0.0:1787695805.303811:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:3.0:1787695805.303816:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000800:00000200:0.0:1787695805.303817:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 6 secs later) 00000100:00000001:3.0:1787695805.303820:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.1:1787695805.303824:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:3.0:1787695805.303825:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000200:0.1:1787695805.303829:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 6 secs later) 00000100:00000001:3.0:1787695805.303830:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695805.303836:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695805.303838:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000800:00000100:0.0:1787695805.303839:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000020:00000001:3.0:1787695805.303841:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000010:0.0:1787695805.303843:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1444fe2500. 00000020:00000040:3.0:1787695805.303844:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000400:00000200:0.0:1787695805.303847:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:3.0:1787695805.303848:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000020:00000001:3.0:1787695805.303852:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000200:0.0:1787695805.303853:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695805.303859:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96c38 00000100:00000040:3.0:1787695805.303859:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000010:0.0:1787695805.303863:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96c38. 00000100:00000001:3.0:1787695805.303864:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695805.303868:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1787695805.303869:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695805.303872:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695805.303876:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00000200:0.0:1787695805.303877:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4a80 x1874531139525120/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695805.303881:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695805.303886:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.303890:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1787695805.303894:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:0.0:1787695805.303895:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4a80 x1874531139525120/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695830 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695805.303898:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695805.303903:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695805.303907:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787695805.303909:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787695805.303910:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787695805.303911:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000001:3.0:1787695805.303915:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.303916:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695805.303917:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695805.303923:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4e00. 00000100:00000040:0.0:1787695805.303924:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000020:00000040:3.0:1787695805.303927:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000400:00000200:0.0:1787695805.303931:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787695805.303933:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787695805.303935:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298366918 02000000:00000001:3.0:1787695805.303935:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695805.303940:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787695805.303941:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298377016 02000000:00000001:3.0:1787695805.303941:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695805.303944:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.303945:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298366918 00000100:00000001:0.0:1787695805.303949:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298377016 02000000:00000010:3.0:1787695805.303949:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f7400. 00000100:00000001:0.0:1787695805.303953:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695805.303954:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695805.303961:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4e00 x1874531139525632/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695805.303963:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.303967:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695805.303973:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695805/real 1787695805] req@ffff8b14466f4a80 x1874531139525120/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695805.303974:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695805.303978:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695805.303985:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcd77cd6] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4e00 x1874531139525632/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695805.303987:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695805.303990:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695805.303992:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bf20. 00000400:00000200:0.0:1787695805.303996:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96d48 00000100:00000001:3.0:1787695805.303997:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695805.303999:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96d48. 00000100:00080000:3.0:1787695805.304001:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695805.304002:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:3.0:1787695805.304007:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000200:0.0:1787695805.304008:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4a80 x1874531139525120/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695805.304009:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695805.304012:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695805.304016:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695805.304021:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3694 00000100:00000200:0.0:1787695805.304025:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4a80 x1874531139525120/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695805.304028:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695805.304034:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695805.304038:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787695805.304039:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695805.304041:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.304042:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695805.304043:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695805.304046:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4a80 x1874531139525120/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695805.304051:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695805.304056:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695805.304058:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695805.304060:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695805.304064:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000040:0.0:1787695805.304065:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4a80 x1874531139525120/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695805.304068:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695805.304070:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000020:00000001:3.0:1787695805.304074:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787695805.304077:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695805.304079:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1787695805.304079:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000100:00000001:0.0:1787695805.304082:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695805.304086:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787695805.304086:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:0.0:1787695805.304089:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695805.304093:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1787695805.304093:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00080000:0.0:1787695805.304099:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000040:3.0:1787695805.304103:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695805.304108:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:0.0:1787695805.304112:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000040:3.0:1787695805.304115:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695805.304116:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787695805.304122:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000040:0.0:1787695805.304123:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4a80 x1874531139525120/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1787695805.304129:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00080000:3.0:1787695805.304137:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00100000:0.0:1787695805.304138:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139525120:192.168.201.140@tcp:38:kworker.0 00000100:00000001:3.0:1787695805.304140:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695805.304144:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787695805.304145:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787695805.304147:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1787695805.304149:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7100 x1874531139525248/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695805.304151:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695805.304155:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695805.304159:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787695805.304161:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:3.0:1787695805.304162:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.304164:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695805.304165:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695805.304167:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695805.304170:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1787695805.304170:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7800. 02000000:00000001:0.0:1787695805.304175:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695805.304175:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=26 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:3.0:1787695805.304181:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:0.0:1787695805.304182:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139525248:192.168.201.140@tcp:38:kworker.0 02000000:00000001:3.0:1787695805.304183:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.304188:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787695805.304188:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695805.304192:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787695805.304192:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787695805.304195:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695805.304196:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695805.304197:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1787695805.304201:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b145101dc00. 02000000:00000010:0.0:1787695805.304202:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695805.304204:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695805.304206:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695805.304211:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bf20. 00000100:00000001:3.0:1787695805.304213:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695805.304215:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96d48. 00000100:00000200:0.0:1787695805.304218:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139525248, portal 10 00000100:00000001:0.0:1787695805.304221:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:3.0:1787695805.304221:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7800 x1874531139525760/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695805.304224:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930156288 : -128555779395328 : ffff8b14466f7100) 00000100:00000001:3.0:1787695805.304232:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:0.0:1787695805.304233:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7100 x1874531139525248/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695805.304236:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695805.304244:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7c57981] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7800 x1874531139525760/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304245:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787695805.304251:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695805.304254:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1787695805.304256:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695805.304259:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000010:0.0:1787695805.304260:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96c38. 00000100:00000040:3.0:1787695805.304265:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:0.0:1787695805.304266:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139525248, offset 0 00000400:00000200:0.0:1787695805.304272:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695805.304272:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695805.304279:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3703) 00000400:00000200:0.0:1787695805.304280:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695805.304287:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695805.304289:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695805.304291:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695805.304293:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695805.304295:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695805.304297:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16913:16913:256:4294967295] 192.168.201.140@tcp LPNI seq info [16912:16912:8:4294967295] 00000400:00000200:0.0:1787695805.304300:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695805.304304:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695805.304307:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695805.304310:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1444fe2500. 00000800:00000200:0.0:1787695805.304312:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695805.304314:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 6 secs later) 00000800:00000200:0.1:1787695805.304318:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695805.304320:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 6 secs later) 00000800:00000100:0.0:1787695805.304324:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695805.304326:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1444fe2500. 00000400:00000200:0.0:1787695805.304329:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695805.304333:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695805.304336:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96c38 00000400:00000010:0.0:1787695805.304338:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96c38. 00000100:00000001:0.0:1787695805.304340:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695805.304345:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7100 x1874531139525248/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304354:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695805.304357:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7100 x1874531139525248/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695830 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304381:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.304383:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695805.304392:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695805.304394:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:0.0:1787695805.304397:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695805.304398:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298365767 00000100:00000001:0.0:1787695805.304400:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298377016 00000100:00000001:0.0:1787695805.304402:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298365767 00000100:00000001:0.0:1787695805.304403:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298377016 00000100:00000001:0.0:1787695805.304405:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695805.304406:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.304408:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695805.304411:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695805/real 1787695805] req@ffff8b14466f7100 x1874531139525248/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695805.304416:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695805.304417:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695805.304418:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bf20. 00000400:00000200:0.0:1787695805.304420:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96d48 00000400:00000010:0.0:1787695805.304421:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96d48. 00000100:00000001:0.0:1787695805.304423:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695805.304425:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7100 x1874531139525248/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695805.304430:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7100 x1874531139525248/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304435:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695805.304436:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.304437:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695805.304438:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695805.304440:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7100 x1874531139525248/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304444:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695805.304446:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7100 x1874531139525248/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304454:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695805.304455:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.304457:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695805.304457:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.304459:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695805.304460:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695805.304461:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695805.304463:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695805.304464:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695805.304467:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7100 x1874531139525248/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695805.304472:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139525248:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695805.304474:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695805.304476:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6680 x1874531139525376/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304481:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695805.304482:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695805.304483:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695805.304484:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695805.304486:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695805.304487:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139525376:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695805.304489:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695805.304491:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695805.304492:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695805.304493:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695805.304496:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450777000. 02000000:00000001:0.0:1787695805.304497:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695805.304500:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695805.304504:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bf20. 00000400:00000010:0.0:1787695805.304507:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96d48. 00000100:00000200:0.0:1787695805.304509:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139525376, portal 4 00000100:00000001:0.0:1787695805.304510:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695805.304511:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930153600 : -128555779398016 : ffff8b14466f6680) 00000100:00000040:0.0:1787695805.304516:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6680 x1874531139525376/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304520:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695805.304521:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695805.304523:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96c38. 00000100:00000200:0.0:1787695805.304524:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139525376, offset 0 00000400:00000200:0.0:1787695805.304527:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695805.304529:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695805.304532:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695805.304534:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695805.304536:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695805.304537:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695805.304539:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695805.304541:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16914:16914:256:4294967295] 192.168.201.140@tcp LPNI seq info [16913:16913:8:4294967295] 00000400:00000200:0.0:1787695805.304544:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695805.304548:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695805.304551:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695805.304553:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1444fe2500. 00000800:00000200:0.0:1787695805.304556:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695805.304558:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 6 secs later) 00000800:00000200:0.1:1787695805.304561:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695805.304562:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 6 secs later) 00000800:00000100:0.0:1787695805.304565:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695805.304566:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1444fe2500. 00000400:00000200:0.0:1787695805.304568:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695805.304571:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695805.304573:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96c38 00000400:00000010:0.0:1787695805.304574:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96c38. 00000100:00000001:0.0:1787695805.304581:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695805.304585:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6680 x1874531139525376/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304594:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695805.304597:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6680 x1874531139525376/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695830 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304604:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.304606:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695805.304610:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695805.304613:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:0.0:1787695805.304616:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695805.304618:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298365797 00000100:00000001:0.0:1787695805.304621:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298377016 00000100:00000001:0.0:1787695805.304623:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298365797 00000100:00000001:0.0:1787695805.304624:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298377016 00000100:00000001:0.0:1787695805.304626:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695805.304627:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.304629:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695805.304632:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695805/real 1787695805] req@ffff8b14466f6680 x1874531139525376/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695805.304637:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695805.304638:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695805.304639:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bf20. 00000400:00000200:0.0:1787695805.304641:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96d48 00000400:00000010:0.0:1787695805.304642:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96d48. 00000100:00000001:0.0:1787695805.304644:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695805.304646:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6680 x1874531139525376/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695805.304651:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6680 x1874531139525376/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304663:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695805.304665:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.304666:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695805.304667:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695805.304670:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6680 x1874531139525376/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304674:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695805.304677:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6680 x1874531139525376/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304681:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695805.304682:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.304683:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695805.304684:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.304685:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695805.304686:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695805.304688:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695805.304690:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695805.304691:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695805.304693:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6680 x1874531139525376/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695805.304698:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139525376:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695805.304700:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.304702:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695805.304704:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4a80 x1874531139525120/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304708:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695805.304709:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695805.304713:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695805.304715:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695805.304716:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695805.304717:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695805.304719:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695805.304720:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f5800. 02000000:00000001:0.0:1787695805.304723:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695805.304724:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695805.304725:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4a80. 00000100:00000001:0.0:1787695805.304727:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695805.304728:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.304729:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695805.304732:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7100 x1874531139525248/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304737:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695805.304739:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695805.304740:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695805.304743:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695805.304744:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695805.304746:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=24 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695805.304748:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695805.304750:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f4400. 02000000:00000001:0.0:1787695805.304753:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695805.304754:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695805.304755:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7100. 00000100:00000001:0.0:1787695805.304758:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695805.304760:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.304761:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695805.304765:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6680 x1874531139525376/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304773:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695805.304774:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695805.304776:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450777000. 02000000:00000001:0.0:1787695805.304778:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695805.304789:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695805.304790:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:0.0:1787695805.304797:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695805.304799:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f5000. 02000000:00000001:0.0:1787695805.304800:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695805.304801:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695805.304802:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6680. 00000100:00000001:0.0:1787695805.304804:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695805.304805:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.304806:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.304809:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695805.304810:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.304811:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695805.304812:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695805.304814:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695805.304817:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5880 x1874531139525504/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304821:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695805.304822:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695805.304824:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695805.304825:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695805.304826:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695805.304828:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139525504:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695805.304830:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695805.304832:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695805.304833:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695805.304834:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695805.304836:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450777000. 02000000:00000001:0.0:1787695805.304837:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695805.304840:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695805.304841:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bf20. 00000400:00000010:0.0:1787695805.304844:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96d48. 00000100:00000200:0.0:1787695805.304846:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139525504, portal 4 00000100:00000001:0.0:1787695805.304847:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695805.304848:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150016 : -128555779401600 : ffff8b14466f5880) 00000100:00000040:0.0:1787695805.304852:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5880 x1874531139525504/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304863:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695805.304865:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695805.304866:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96c38. 00000100:00000200:0.0:1787695805.304868:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139525504, offset 0 00000400:00000200:0.0:1787695805.304871:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695805.304874:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695805.304876:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695805.304878:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695805.304880:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695805.304882:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695805.304883:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695805.304886:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16915:16915:256:4294967295] 192.168.201.140@tcp LPNI seq info [16914:16914:8:4294967295] 00000400:00000200:0.0:1787695805.304888:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695805.304893:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695805.304895:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695805.304897:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1444fe2500. 00000800:00000200:0.0:1787695805.304900:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695805.304902:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 6 secs later) 00000800:00000200:0.1:1787695805.304904:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695805.304906:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 6 secs later) 00000800:00000100:0.0:1787695805.304909:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695805.304910:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1444fe2500. 00000400:00000200:0.0:1787695805.304912:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695805.304914:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695805.304916:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96c38 00000400:00000010:0.0:1787695805.304917:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96c38. 00000100:00000001:0.0:1787695805.304919:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695805.304921:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5880 x1874531139525504/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304926:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695805.304928:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5880 x1874531139525504/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695830 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304940:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.304941:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695805.304944:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695805.304946:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:0.0:1787695805.304948:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695805.304949:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298365469 00000100:00000001:0.0:1787695805.304951:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298377016 00000100:00000001:0.0:1787695805.304953:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298365469 00000100:00000001:0.0:1787695805.304954:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298377016 00000100:00000001:0.0:1787695805.304955:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695805.304957:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.304958:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695805.304960:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695805/real 1787695805] req@ffff8b14466f5880 x1874531139525504/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695805.304965:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695805.304966:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695805.304967:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bf20. 00000400:00000200:0.0:1787695805.304969:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96d48 00000400:00000010:0.0:1787695805.304970:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96d48. 00000100:00000001:0.0:1787695805.304971:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695805.304973:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5880 x1874531139525504/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695805.304979:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5880 x1874531139525504/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304982:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695805.304983:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.304984:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695805.304985:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695805.304987:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5880 x1874531139525504/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.304995:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695805.304998:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5880 x1874531139525504/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.305002:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695805.305004:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.305005:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695805.305005:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.305006:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695805.305007:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695805.305009:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695805.305011:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695805.305012:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695805.305014:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5880 x1874531139525504/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695805.305019:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139525504:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695805.305022:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695805.305024:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4e00 x1874531139525632/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.305028:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695805.305029:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695805.305030:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695805.305031:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695805.305032:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695805.305034:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139525632:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695805.305036:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695805.305037:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695805.305041:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695805.305043:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695805.305046:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695805.305048:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695805.305051:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695805.305054:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bf20. 00000400:00000010:0.0:1787695805.305057:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96d48. 00000100:00000200:0.0:1787695805.305060:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139525632, portal 10 00000100:00000001:0.0:1787695805.305063:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695805.305064:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930147328 : -128555779404288 : ffff8b14466f4e00) 00000100:00000040:0.0:1787695805.305070:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4e00 x1874531139525632/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.305077:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695805.305079:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695805.305082:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96c38. 00000100:00000200:0.0:1787695805.305085:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139525632, offset 0 00000400:00000200:0.0:1787695805.305088:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695805.305092:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695805.305095:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695805.305098:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695805.305101:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695805.305103:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695805.305105:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695805.305109:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16916:16916:256:4294967295] 192.168.201.140@tcp LPNI seq info [16915:16915:8:4294967295] 00000400:00000200:0.0:1787695805.305113:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695805.305119:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695805.305123:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695805.305126:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1444fe2500. 00000800:00000200:0.0:1787695805.305130:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695805.305132:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 6 secs later) 00000800:00000200:0.1:1787695805.305136:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695805.305139:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 6 secs later) 00000800:00000100:0.0:1787695805.305143:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695805.305150:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1444fe2500. 00000400:00000200:0.0:1787695805.305153:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695805.305157:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695805.305161:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96c38 00000400:00000010:0.0:1787695805.305162:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96c38. 00000100:00000001:0.0:1787695805.305164:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695805.305167:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4e00 x1874531139525632/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.305173:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695805.305175:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4e00 x1874531139525632/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695830 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.305179:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.305180:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695805.305182:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695805.305185:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:0.0:1787695805.305187:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695805.305188:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298365655 00000100:00000001:0.0:1787695805.305190:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298377016 00000100:00000001:0.0:1787695805.305191:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298365655 00000100:00000001:0.0:1787695805.305193:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298377016 00000100:00000001:0.0:1787695805.305195:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695805.305196:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.305197:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695805.305200:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695805/real 1787695805] req@ffff8b14466f4e00 x1874531139525632/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695805.305204:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695805.305205:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695805.305206:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bf20. 00000400:00000200:0.0:1787695805.305208:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96d48 00000400:00000010:0.0:1787695805.305209:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96d48. 00000100:00000001:0.0:1787695805.305214:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695805.305217:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4e00 x1874531139525632/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695805.305222:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4e00 x1874531139525632/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.305226:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695805.305227:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.305228:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695805.305229:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695805.305231:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4e00 x1874531139525632/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.305235:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695805.305238:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4e00 x1874531139525632/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.305242:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695805.305243:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.305244:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695805.305245:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.305246:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695805.305247:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695805.305248:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695805.305250:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695805.305251:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695805.305254:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4e00 x1874531139525632/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695805.305258:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139525632:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695805.305261:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695805.305267:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7800 x1874531139525760/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.305271:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695805.305272:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695805.305274:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695805.305275:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695805.305277:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695805.305278:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139525760:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695805.305281:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695805.305282:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695805.305283:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695805.305284:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695805.305286:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695805.305287:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695805.305290:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695805.305291:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bf20. 00000400:00000010:0.0:1787695805.305293:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96d48. 00000100:00000200:0.0:1787695805.305295:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139525760, portal 10 00000100:00000001:0.0:1787695805.305297:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695805.305298:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930158080 : -128555779393536 : ffff8b14466f7800) 00000100:00000040:0.0:1787695805.305302:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7800 x1874531139525760/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.305306:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695805.305307:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695805.305309:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96c38. 00000100:00000200:0.0:1787695805.305310:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139525760, offset 0 00000400:00000200:0.0:1787695805.305313:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695805.305315:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695805.305318:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695805.305321:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695805.305324:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695805.305326:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695805.305334:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695805.305337:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16917:16917:256:4294967295] 192.168.201.140@tcp LPNI seq info [16916:16916:8:4294967295] 00000400:00000200:0.0:1787695805.305341:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695805.305348:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695805.305353:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695805.305356:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1444fe2500. 00000800:00000200:0.0:1787695805.305361:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695805.305379:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 6 secs later) 00000800:00000200:0.1:1787695805.305383:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695805.305385:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 6 secs later) 00000800:00000100:0.0:1787695805.305389:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695805.305391:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1444fe2500. 00000400:00000200:0.0:1787695805.305394:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695805.305398:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695805.305400:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96c38 00000400:00000010:0.0:1787695805.305402:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96c38. 00000100:00000001:0.0:1787695805.305405:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695805.305408:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7800 x1874531139525760/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695830 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.305414:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695805.305417:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7800 x1874531139525760/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695830 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.305422:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.305424:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695805.305426:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695805.305428:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:0.0:1787695805.305430:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695805.305432:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298365634 00000100:00000001:0.0:1787695805.305434:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298377016 00000100:00000001:0.0:1787695805.305435:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298365634 00000100:00000001:0.0:1787695805.305437:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298377016 00000100:00000001:0.0:1787695805.305442:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695805.305444:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.305445:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695805.305448:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695805/real 1787695805] req@ffff8b14466f7800 x1874531139525760/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695805.305454:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695805.305454:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695805.305456:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bf20. 00000400:00000200:0.0:1787695805.305457:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96d48 00000400:00000010:0.0:1787695805.305458:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96d48. 00000100:00000001:0.0:1787695805.305460:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695805.305462:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7800 x1874531139525760/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695805.305467:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7800 x1874531139525760/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.305471:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695805.305471:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.305472:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695805.305473:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695805.305475:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7800 x1874531139525760/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.305479:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695805.305482:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7800 x1874531139525760/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.305486:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695805.305488:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.305489:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695805.305490:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.305491:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695805.305492:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695805.305497:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695805.305499:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695805.305500:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695805.305503:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7800 x1874531139525760/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695805.305508:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139525760:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695805.305510:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.305511:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695805.305513:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5880 x1874531139525504/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.305517:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695805.305519:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695805.305520:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450777000. 02000000:00000001:0.0:1787695805.305522:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695805.305523:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695805.305524:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:0.0:1787695805.305525:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695805.305526:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f5400. 02000000:00000001:0.0:1787695805.305528:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695805.305529:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695805.305530:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5880. 00000100:00000001:0.0:1787695805.305532:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695805.305533:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.305534:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695805.305536:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4e00 x1874531139525632/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.305540:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695805.305540:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695805.305541:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695805.305542:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695805.305543:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695805.305546:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695805.305548:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695805.305549:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f7400. 02000000:00000001:0.0:1787695805.305550:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695805.305551:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695805.305552:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4e00. 00000100:00000001:0.0:1787695805.305553:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695805.305554:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.305555:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695805.305557:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7800 x1874531139525760/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695830 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695805.305561:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695805.305562:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695805.305563:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695805.305564:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695805.305564:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695805.305565:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=25 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695805.305566:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695805.305567:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b145101dc00. 02000000:00000001:0.0:1787695805.305568:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695805.305569:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695805.305570:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7800. 00000100:00000001:0.0:1787695805.305571:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695805.305572:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.305573:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695805.305576:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695805.305577:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695805.305579:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695805.305580:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695805.366467:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695805.366477:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695805.366479:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695805.366485:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695805.366489:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695805.366492:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695805.366495:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695805.366497:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695805.366500:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695806.006513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695806.006519:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695806.006523:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695806.006524:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695806.006527:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695806.006528:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695806.006531:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695806.006531:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695806.006533:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695806.006534:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695806.006536:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695806.006537:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695806.006597:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695806.006602:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695806.006606:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695806.390505:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695806.390517:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695806.390520:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695806.390527:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695806.390532:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695806.390537:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695806.390541:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695806.390545:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695806.390548:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695806.838481:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695806.838495:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695806.838497:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695806.838502:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695806.838504:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695806.838512:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695806.838515:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695807.030440:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695807.030446:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695807.030451:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695807.030452:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695807.030454:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695807.030457:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695807.030459:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695807.030464:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695807.030465:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695807.030466:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695807.030469:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695807.030471:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695807.030472:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695807.030474:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695807.030491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695807.030493:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695807.030494:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695807.030495:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695807.030496:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695807.030497:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695807.030498:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695807.030499:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695807.030500:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695807.030501:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695807.030504:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695807.030505:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695807.030507:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695807.030508:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695807.030509:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695807.030510:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695807.030510:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695807.030511:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695807.030512:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695807.030513:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695807.030514:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695807.030515:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695807.030569:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695807.030573:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695807.030577:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695807.414436:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695807.414442:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695807.414444:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695807.414449:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695807.414452:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695807.414455:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695807.414457:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695807.414459:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695807.414460:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695807.862420:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695807.862438:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695807.862440:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695807.862444:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695807.862445:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695807.862450:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695807.862452:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695808.054470:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695808.054479:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695808.054483:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695808.054484:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695808.054487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695808.054488:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695808.054492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695808.054501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695808.054504:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695808.054506:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695808.054508:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695808.054509:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695808.054570:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695808.054577:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695808.054581:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695808.438467:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695808.438473:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695808.438475:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695808.438480:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695808.438482:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695808.438486:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695808.438488:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695808.438490:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695808.438492:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695808.886528:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695808.886544:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695808.886546:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695808.886552:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695808.886578:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695808.886585:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695808.886589:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695809.078494:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695809.078503:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695809.078507:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695809.078508:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695809.078511:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695809.078533:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695809.078537:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695809.078538:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695809.078541:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695809.078542:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695809.078545:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695809.078546:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695809.078631:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695809.078636:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695809.078648:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695809.462451:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695809.462463:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695809.462466:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695809.462482:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695809.462486:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695809.462490:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695809.462493:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695809.462497:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695809.462501:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695809.910419:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695809.910426:0:2406:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000400:3.0:1787695809.910440:0:2406:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695724/real 1787695724] req@ffff8b1446605f80 x1874531139513472/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695809 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787695809.910455:0:2406:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787695809.910458:0:2406:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1787695809.910460:0:2406:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bb00. 00000400:00000200:3.0:1787695809.910467:0:2406:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8330 00000400:00000010:3.0:1787695809.910469:0:2406:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8330. 00000100:00000001:3.0:1787695809.910473:0:2406:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787695809.910477:0:2406:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446605f80 x1874531139513472/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695809 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1787695809.910484:0:2406:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446605f80 x1874531139513472/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695809 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695809.910489:0:2406:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787695809.910490:0:2406:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695809.910493:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695809.910494:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787695809.910498:0:2406:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446605f80 x1874531139513472/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695809 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695809.910503:0:2406:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:3.0:1787695809.910506:0:2406:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b144515c000 already not connected (conn 16, was 11): DISCONN 00000100:00000001:3.0:1787695809.910521:0:2406:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:3.0:1787695809.910522:0:2406:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695809.910524:0:2406:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000400:3.0:1787695809.910527:0:2406:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695724/real 1787695724] req@ffff8b1446606a00 x1874531139512960/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695809 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787695809.910534:0:2406:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787695809.910535:0:2406:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1787695809.910536:0:2406:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752b2c0. 00000400:00000200:3.0:1787695809.910538:0:2406:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8660 00000400:00000010:3.0:1787695809.910540:0:2406:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8660. 00000100:00000001:3.0:1787695809.910542:0:2406:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787695809.910545:0:2406:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446606a00 x1874531139512960/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695809 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1787695809.910551:0:2406:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446606a00 x1874531139512960/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695809 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695809.910556:0:2406:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787695809.910556:0:2406:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695809.910557:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695809.910558:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787695809.910561:0:2406:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446606a00 x1874531139512960/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695809 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695809.910565:0:2406:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:3.0:1787695809.910567:0:2406:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b1458081800 already not connected (conn 20, was 15): DISCONN 00000100:00000001:3.0:1787695809.910570:0:2406:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:3.0:1787695809.910572:0:2406:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695809.910580:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695809.910583:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695809.910586:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695809.910590:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695809.910597:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695809.910600:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695809.910603:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695809.910616:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446605f80 x1874531139513472/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695809 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695809.910626:0:2406:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:3.0:1787695809.910630:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695809.910632:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695809.910634:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695809.910640:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446605f80 x1874531139513472/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695809 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695809.910650:0:2406:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446605f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:608c526c-eb6e-4a54-80a5-8969828241eb:2406:1874531139513472:192.168.201.140@tcp:400:kworker.0 00000100:00000001:3.0:1787695809.910656:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695809.910658:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695809.910660:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695809.910664:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446606a00 x1874531139512960/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695809 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695809.910747:0:2406:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:3.0:1787695809.910751:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695809.910753:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695809.910755:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695809.910761:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446606a00 x1874531139512960/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695809 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695809.910770:0:2406:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446606a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2406:1874531139512960:192.168.201.140@tcp:400:kworker.0 00000100:00000001:3.0:1787695809.910775:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695809.910778:0:2406:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695809.910782:0:2406:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446605f80 x1874531139513472/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695809 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695809.910789:0:2406:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787695809.910792:0:2406:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1787695809.910804:0:2406:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1447533000. 02000000:00000001:3.0:1787695809.910809:0:2406:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787695809.910812:0:2406:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1787695809.910815:0:2406:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=24 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:3.0:1787695809.910817:0:2406:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1787695809.910820:0:2406:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1458095000. 02000000:00000001:3.0:1787695809.910825:0:2406:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787695809.910827:0:2406:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1787695809.910829:0:2406:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446605f80. 00000100:00000001:3.0:1787695809.910833:0:2406:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787695809.910834:0:2406:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695809.910836:0:2406:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695809.910840:0:2406:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446606a00 x1874531139512960/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695809 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695809.910844:0:2406:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787695809.910953:0:2406:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1787695809.910956:0:2406:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1445def000. 02000000:00000001:3.0:1787695809.910959:0:2406:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787695809.910960:0:2406:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1787695809.910961:0:2406:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=23 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:3.0:1787695809.910962:0:2406:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1787695809.910964:0:2406:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1458095900. 02000000:00000001:3.0:1787695809.910965:0:2406:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787695809.910972:0:2406:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1787695809.910974:0:2406:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446606a00. 00000100:00000001:3.0:1787695809.910976:0:2406:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787695809.911010:0:2406:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695809.911012:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695810.102474:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695810.102481:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695810.102487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695810.102489:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695810.102492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695810.102494:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695810.102497:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695810.102498:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695810.102502:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695810.102503:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695810.102506:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695810.102507:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695810.102573:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695810.102578:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695810.102581:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787695810.422494:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695810.422504:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787695810.422507:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695810.422511:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695810.422515:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695810.422522:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3698 00000400:00000200:1.0:1787695810.422531:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695810.422537:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695810.422540:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695810.422543:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695810.422547:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695810.422550:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695810.422552:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695810.422556:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695810.422558:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:1.0:1787695810.422569:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695810.422572:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:1.0:1787695810.422575:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:1.0:1787695810.422578:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:1.0:1787695810.422581:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695810.422584:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695810.422586:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695810.422588:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695810.422591:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:1.0:1787695810.422594:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695810.422596:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695810.422599:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695810.422601:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695810.422605:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695810.422607:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695810.422610:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695810.422616:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446666300. 00000020:00000040:1.0:1787695810.422620:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:1.0:1787695810.422624:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695810.422626:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695810.422629:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695810.422631:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695810.422636:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695810.422645:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446547800. 00000100:00000001:1.0:1787695810.422652:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695810.422662:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446666300 x1874531139525888/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695810.422673:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695810.422678:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695810.422683:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000945f4e20] to pc [ptlrpcd_rcv+-1] req@ffff8b1446666300 x1874531139525888/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695810.422722:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695810.422743:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695810.422747:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787695810.422749:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695810.422751:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695810.422755:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695810.422756:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695810.422761:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3698 00000100:00000001:0.0:1787695810.422763:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695810.422768:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000400:00000200:1.0:1787695810.422768:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695810.422775:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000040:0.0:1787695810.422780:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446666300 x1874531139525888/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695810.422784:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695810.422792:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695810.422797:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695810.422801:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1787695810.422803:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695810.422805:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695810.422808:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:0.0:1787695810.422810:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1787695810.422814:0:38:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:0.0:1787695810.422815:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1787695810.422817:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 02000000:00000001:0.0:1787695810.422821:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787695810.422822:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695810.422828:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000100:00100000:0.0:1787695810.422831:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446666300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139525888:192.168.201.140@tcp:250:kworker.0 00000020:00000001:1.0:1787695810.422833:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:1.0:1787695810.422838:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000001:0.0:1787695810.422841:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1787695810.422845:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695810.422847:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1787695810.422849:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787695810.422852:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695810.422854:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695810.422858:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787695810.422859:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000010:0.0:1787695810.422864:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450777000. 00000020:00000040:1.0:1787695810.422864:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 02000000:00000001:0.0:1787695810.422869:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695810.422870:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695810.422875:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695810.422876:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:1.0:1787695810.422880:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:0.0:1787695810.422882:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bf20. 02000000:00000001:1.0:1787695810.422887:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1787695810.422888:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96d48. 02000000:00000001:1.0:1787695810.422894:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695810.422897:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000200:0.0:1787695810.422898:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139525888, portal 25 00000100:00000001:0.0:1787695810.422901:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695810.422902:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929562880 : -128555779988736 : ffff8b1446666300) 00000008:00100000:1.0:1787695810.422909:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000040:0.0:1787695810.422910:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446666300 x1874531139525888/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1787695810.422918:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.422923:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787695810.422924:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695810.422928:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695810.422932:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96c38. 00000100:00000010:1.0:1787695810.422932:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446664e00. 00000100:00000200:0.0:1787695810.422937:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139525888, offset 0 00000020:00000040:1.0:1787695810.422939:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000400:00000200:0.0:1787695810.422944:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 02000000:00000001:1.0:1787695810.422944:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695810.422947:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695810.422958:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695810.422960:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695810.422964:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:0.0:1787695810.422965:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 02000000:00000001:1.0:1787695810.422968:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695810.422971:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695810.422976:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 02000000:00000010:1.0:1787695810.422976:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446547c00. 00000400:00000200:0.0:1787695810.422981:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1787695810.422983:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695810.422984:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695810.422989:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16918:16918:256:4294967295] 192.168.201.140@tcp LPNI seq info [16917:16917:8:4294967295] 00000400:00000200:0.0:1787695810.422995:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00100000:1.0:1787695810.422995:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446664e00 x1874531139526016/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1787695810.423010:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:1.0:1787695810.423012:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:0.0:1787695810.423019:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787695810.423019:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1787695810.423027:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1444fe2500. 00000100:00000040:1.0:1787695810.423029:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b012f68f] to pc [ptlrpcd_rcv+-1] req@ffff8b1446664e00 x1874531139526016/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1787695810.423035:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695810.423041:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 1 secs later) 00000800:00000200:0.1:1787695810.423048:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:1.0:1787695810.423050:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1787695810.423052:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 1 secs later) 00000100:00080000:1.0:1787695810.423056:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787695810.423058:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695810.423062:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1444fe2500. 00000100:00080000:1.0:1787695810.423065:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:0.0:1787695810.423067:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695810.423070:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695810.423074:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96c38 00000400:00000010:0.0:1787695810.423076:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96c38. 00000100:00000001:1.0:1787695810.423078:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787695810.423080:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:1.0:1787695810.423082:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:0.0:1787695810.423085:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446666300 x1874531139525888/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695810.423089:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695810.423096:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3698 00000100:00000001:0.0:1787695810.423098:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695810.423104:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446666300 x1874531139525888/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695835 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695810.423105:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695810.423114:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695810.423117:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695810.423120:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695810.423122:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695810.423124:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695810.423125:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1787695810.423129:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695810.423132:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695810.423136:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695810.423140:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298382136 00000100:00000001:1.0:1787695810.423142:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695810.423148:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298377016 00000020:00000001:1.0:1787695810.423156:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787695810.423159:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298382136 00000020:00000040:1.0:1787695810.423160:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000100:00000001:0.0:1787695810.423164:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298377016 00000020:00000001:1.0:1787695810.423165:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787695810.423170:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1787695810.423170:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000100:00000001:0.0:1787695810.423177:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787695810.423177:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000001:0.0:1787695810.423183:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:1.0:1787695810.423184:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00100000:0.0:1787695810.423192:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695810/real 1787695810] req@ffff8b1446666300 x1874531139525888/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695810.423193:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695810.423197:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695810.423201:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695810.423204:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695810.423206:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:1.0:1787695810.423211:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695810.423217:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695810.423221:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:0.0:1787695810.423223:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1787695810.423226:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:0.0:1787695810.423228:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:1.0:1787695810.423232:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695810.423233:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bf20. 00000100:00000001:1.0:1787695810.423237:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:0.0:1787695810.423240:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96d48 00000008:00100000:1.0:1787695810.423242:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000010:0.0:1787695810.423244:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96d48. 00000008:00000001:1.0:1787695810.423246:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.423250:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787695810.423251:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695810.423258:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446664380. 00000100:00000200:0.0:1787695810.423259:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446666300 x1874531139525888/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1787695810.423263:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=24 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:1.0:1787695810.423266:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695810.423267:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695810.423269:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695810.423271:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695810.423275:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695810.423281:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545c00. 00000100:00000200:0.0:1787695810.423287:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446666300 x1874531139525888/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695810.423287:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695810.423297:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446664380 x1874531139526144/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695810.423301:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695810.423305:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.423309:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695810.423311:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695810.423321:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446666300 x1874531139525888/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695810.423326:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695810.423331:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.423338:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695810.423341:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000094a36b85] to pc [ptlrpcd_rcv+-1] req@ffff8b1446664380 x1874531139526144/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695810.423348:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446666300 x1874531139525888/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695810.423361:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.423393:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695810.423397:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.423400:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:1.0:1787695810.423406:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695810.423409:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787695810.423412:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787695810.423415:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695810.423416:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787695810.423419:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695810.423420:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787695810.423427:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1787695810.423427:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695810.423434:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3698 00000100:00080000:0.0:1787695810.423437:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000400:00000200:1.0:1787695810.423442:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695810.423443:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000400:00000200:1.0:1787695810.423450:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000040:0.0:1787695810.423454:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446666300 x1874531139525888/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695810.423455:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695810.423461:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695810.423464:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695810.423467:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695810.423470:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695810.423474:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695810.423476:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:1.0:1787695810.423479:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695810.423482:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000100:00100000:0.0:1787695810.423483:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446666300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139525888:192.168.201.140@tcp:250:kworker.0 00000020:00000001:1.0:1787695810.423484:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:1.0:1787695810.423488:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:1.0:1787695810.423493:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695810.423495:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695810.423495:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695810.423499:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695810.423501:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787695810.423502:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000040:0.0:1787695810.423508:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446666300 x1874531139525888/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1787695810.423512:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:1.0:1787695810.423519:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695810.423524:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695810.423527:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787695810.423528:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695810.423531:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695810.423532:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787695810.423534:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695810.423537:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000010:0.0:1787695810.423538:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450777000. 00000008:00100000:1.0:1787695810.423539:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787695810.423544:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695810.423546:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1787695810.423546:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695810.423549:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446666d80. 00000020:00000040:1.0:1787695810.423551:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:1.0:1787695810.423552:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695810.423553:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695810.423554:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695810.423556:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:0.0:1787695810.423559:0:2402:0:(genops.c:1217:class_import_put()) Process entered 02000000:00000001:1.0:1787695810.423559:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1787695810.423563:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 02000000:00000010:1.0:1787695810.423565:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446546800. 00000020:00000001:0.0:1787695810.423569:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:1.0:1787695810.423570:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695810.423575:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446547800. 00000100:00100000:1.0:1787695810.423577:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446666d80 x1874531139526272/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787695810.423582:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1787695810.423585:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1787695810.423586:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787695810.423589:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1787695810.423591:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446666300. 00000100:00000040:1.0:1787695810.423594:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa82a36] to pc [ptlrpcd_rcv+-1] req@ffff8b1446666d80 x1874531139526272/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.423597:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695810.423601:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.423607:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695810.423607:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695810.423612:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695810.423615:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1787695810.423616:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787695810.423618:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1787695810.423618:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695810.423622:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695810.423623:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787695810.423627:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1787695810.423630:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3698 00000100:00000001:0.0:1787695810.423632:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695810.423636:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000400:00000200:1.0:1787695810.423636:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695810.423641:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695810.423644:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1787695810.423646:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446664e00 x1874531139526016/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695810.423648:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695810.423649:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695810.423651:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695810.423652:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695810.423656:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695810.423658:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:1.0:1787695810.423660:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695810.423661:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:1.0:1787695810.423665:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:1.0:1787695810.423667:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:1.0:1787695810.423673:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695810.423676:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695810.423677:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1787695810.423678:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695810.423681:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695810.423681:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695810.423687:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 02000000:00000001:0.0:1787695810.423688:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:1.0:1787695810.423704:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000001:0.0:1787695810.423706:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695810.423708:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695810.423712:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695810.423712:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695810.423716:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695810.423720:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695810.423721:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446664e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139526016:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695810.423722:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787695810.423725:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787695810.423730:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.423731:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787695810.423733:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695810.423735:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:1.0:1787695810.423737:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665180. 02000000:00000001:0.0:1787695810.423739:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1787695810.423740:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:0.0:1787695810.423746:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787695810.423746:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000010:0.0:1787695810.423753:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450777000. 02000000:00000001:1.0:1787695810.423753:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695810.423757:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695810.423759:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695810.423759:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695810.423762:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695810.423767:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000010:1.0:1787695810.423768:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545800. 00000100:00000001:1.0:1787695810.423773:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695810.423774:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bf20. 00000100:00100000:1.0:1787695810.423779:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446665180 x1874531139526400/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:0.0:1787695810.423782:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96d48. 00000100:00000200:0.0:1787695810.423789:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139526016, portal 10 00000100:00000001:1.0:1787695810.423789:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695810.423793:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.423795:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:1.0:1787695810.423798:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f923372e] to pc [ptlrpcd_rcv+-1] req@ffff8b1446665180 x1874531139526400/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.423799:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929557504 : -128555779994112 : ffff8b1446664e00) 00000100:00000001:1.0:1787695810.423810:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695810.423812:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446664e00 x1874531139526016/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695810.423813:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695810.423820:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787695810.423822:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695810.423825:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695810.423829:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695810.423833:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3698 00000100:00000001:0.0:1787695810.423834:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:1.0:1787695810.423838:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:0.0:1787695810.423843:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695810.423844:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695810.423847:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000400:00000010:0.0:1787695810.423849:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96c38. 00000100:00000040:1.0:1787695810.423852:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000200:0.0:1787695810.423855:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139526016, offset 0 00000100:00000001:1.0:1787695810.423855:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695810.423859:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695810.423862:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695810.423864:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:0.0:1787695810.423867:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000020:00000040:1.0:1787695810.423867:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:1.0:1787695810.423870:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695810.423874:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000020:00000001:1.0:1787695810.423877:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000200:0.0:1787695810.423878:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:1.0:1787695810.423883:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000200:0.0:1787695810.423885:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000040:1.0:1787695810.423890:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695810.423892:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1787695810.423893:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1787695810.423899:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000040:1.0:1787695810.423899:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695810.423904:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695810.423907:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000400:00000200:0.0:1787695810.423908:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00080000:1.0:1787695810.423912:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:0.0:1787695810.423916:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:1.0:1787695810.423916:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695810.423919:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695810.423921:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695810.423922:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695810.423923:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787695810.423925:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787695810.423928:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695810.423931:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695810.423932:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16919:16919:256:4294967295] 192.168.201.140@tcp LPNI seq info [16918:16918:8:4294967295] 00000100:00000010:1.0:1787695810.423935:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446666680. 00000020:00000040:1.0:1787695810.423938:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:1.0:1787695810.423942:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695810.423944:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695810.423945:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695810.423946:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695810.423948:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695810.423952:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695810.423955:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545400. 00000100:00000001:1.0:1787695810.423960:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695810.423965:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00100000:1.0:1787695810.423968:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446666680 x1874531139526528/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.0:1787695810.423977:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787695810.423977:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695810.423979:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1787695810.423985:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1444fe2500. 00000100:00000040:1.0:1787695810.423985:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bd4d567] to pc [ptlrpcd_rcv+-1] req@ffff8b1446666680 x1874531139526528/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695810.423994:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1787695810.423996:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695810.424001:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 1 secs later) 00000100:00080000:1.0:1787695810.424002:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695810.424009:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787695810.424012:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695810.424014:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:0.1:1787695810.424014:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:1.0:1787695810.424021:0:38:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:0.1:1787695810.424022:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 1 secs later) 00000100:00080000:1.0:1787695810.424024:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3698 00000400:00000200:1.0:1787695810.424030:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000800:00000100:0.0:1787695810.424034:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695810.424035:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000800:00000010:0.0:1787695810.424039:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1444fe2500. 00000100:00080000:1.0:1787695810.424039:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695810.424044:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695810.424048:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1787695810.424048:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695810.424051:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695810.424054:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695810.424057:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695810.424060:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000400:00000200:0.0:1787695810.424061:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000020:00000001:1.0:1787695810.424062:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695810.424064:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:1.0:1787695810.424068:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000400:00000200:0.0:1787695810.424070:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96c38 00000020:00000001:1.0:1787695810.424070:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000400:00000010:0.0:1787695810.424075:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96c38. 00000100:00000040:1.0:1787695810.424076:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695810.424080:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695810.424084:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:1.0:1787695810.424084:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695810.424088:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695810.424090:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00000200:0.0:1787695810.424094:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446664e00 x1874531139526016/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695810.424095:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695810.424098:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695810.424101:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695810.424104:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695810.424108:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695810.424111:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787695810.424117:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787695810.424118:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000008:00000001:1.0:1787695810.424122:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695810.424125:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695810.424127:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446664e00 x1874531139526016/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695835 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1787695810.424128:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446664000. 00000020:00000040:1.0:1787695810.424132:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=25 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:1.0:1787695810.424135:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695810.424137:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695810.424140:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695810.424142:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695810.424144:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.424149:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695810.424150:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446547400. 00000100:00000001:0.0:1787695810.424154:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695810.424155:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.424161:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695810.424162:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446664000 x1874531139526656/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1787695810.424169:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000100:00000001:1.0:1787695810.424170:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695810.424173:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695810.424178:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e60c2a2d] to pc [ptlrpcd_rcv+-1] req@ffff8b1446664000 x1874531139526656/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695810.424179:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695810.424185:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298372038 00000100:00000001:1.0:1787695810.424187:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.424192:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298382136 00000100:00000040:1.0:1787695810.424193:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695810.424200:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298372038 00000100:00000040:1.0:1787695810.424201:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:1.0:1787695810.424205:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695810.424208:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298382136 00000100:00000040:1.0:1787695810.424208:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1787695810.424213:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1787695810.424216:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3708) 00000100:00000001:0.0:1787695810.424222:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.424224:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695810.424229:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695810/real 1787695810] req@ffff8b1446664e00 x1874531139526016/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695810.424236:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695810.424238:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695810.424239:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bf20. 00000400:00000200:0.0:1787695810.424241:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96d48 00000400:00000010:0.0:1787695810.424242:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96d48. 00000100:00000001:0.0:1787695810.424244:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695810.424246:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446664e00 x1874531139526016/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695810.424252:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446664e00 x1874531139526016/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424256:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695810.424257:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.424258:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695810.424259:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695810.424262:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446664e00 x1874531139526016/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424266:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695810.424268:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446664e00 x1874531139526016/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424273:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695810.424275:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.424276:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695810.424276:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.424278:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695810.424279:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695810.424281:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695810.424286:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695810.424288:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695810.424291:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446664e00 x1874531139526016/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695810.424296:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446664e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139526016:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695810.424298:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695810.424300:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446664380 x1874531139526144/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424304:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695810.424306:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695810.424307:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695810.424309:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695810.424310:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695810.424312:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446664380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139526144:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695810.424314:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695810.424315:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695810.424316:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695810.424317:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695810.424320:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695810.424321:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695810.424324:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695810.424326:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bf20. 00000400:00000010:0.0:1787695810.424328:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96d48. 00000100:00000200:0.0:1787695810.424330:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139526144, portal 10 00000100:00000001:0.0:1787695810.424331:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695810.424332:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929554816 : -128555779996800 : ffff8b1446664380) 00000100:00000040:0.0:1787695810.424336:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446664380 x1874531139526144/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424341:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695810.424346:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695810.424347:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96c38. 00000100:00000200:0.0:1787695810.424349:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139526144, offset 0 00000400:00000200:0.0:1787695810.424352:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695810.424355:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695810.424358:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695810.424360:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695810.424376:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695810.424378:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695810.424380:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695810.424384:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16920:16920:256:4294967295] 192.168.201.140@tcp LPNI seq info [16919:16919:8:4294967295] 00000400:00000200:0.0:1787695810.424388:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695810.424393:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695810.424397:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695810.424399:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1444fe2500. 00000800:00000200:0.0:1787695810.424402:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695810.424404:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 1 secs later) 00000800:00000200:0.1:1787695810.424407:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695810.424409:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 1 secs later) 00000800:00000100:0.0:1787695810.424411:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695810.424413:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1444fe2500. 00000400:00000200:0.0:1787695810.424415:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695810.424417:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695810.424419:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96c38 00000400:00000010:0.0:1787695810.424421:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96c38. 00000100:00000001:0.0:1787695810.424422:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695810.424425:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446664380 x1874531139526144/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424430:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695810.424432:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446664380 x1874531139526144/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695835 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424437:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.424438:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695810.424444:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695810.424447:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:0.0:1787695810.424449:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695810.424451:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298370887 00000100:00000001:0.0:1787695810.424452:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298382136 00000100:00000001:0.0:1787695810.424454:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298370887 00000100:00000001:0.0:1787695810.424455:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298382136 00000100:00000001:0.0:1787695810.424457:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695810.424458:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.424459:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695810.424462:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695810/real 1787695810] req@ffff8b1446664380 x1874531139526144/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695810.424467:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695810.424468:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695810.424469:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bf20. 00000400:00000200:0.0:1787695810.424471:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96d48 00000400:00000010:0.0:1787695810.424472:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96d48. 00000100:00000001:0.0:1787695810.424473:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695810.424475:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446664380 x1874531139526144/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695810.424480:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446664380 x1874531139526144/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424484:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695810.424485:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.424486:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695810.424487:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695810.424489:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446664380 x1874531139526144/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424493:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695810.424495:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446664380 x1874531139526144/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424503:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695810.424505:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.424506:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695810.424507:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.424508:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695810.424509:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695810.424511:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695810.424513:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695810.424514:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695810.424517:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446664380 x1874531139526144/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695810.424522:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446664380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139526144:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695810.424524:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695810.424526:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446666d80 x1874531139526272/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424530:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695810.424531:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695810.424532:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695810.424533:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695810.424535:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695810.424536:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446666d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139526272:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695810.424538:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695810.424540:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695810.424540:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695810.424542:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695810.424544:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695810.424545:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695810.424548:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695810.424554:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bf20. 00000400:00000010:0.0:1787695810.424556:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96d48. 00000100:00000200:0.0:1787695810.424559:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139526272, portal 4 00000100:00000001:0.0:1787695810.424560:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695810.424561:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929565568 : -128555779986048 : ffff8b1446666d80) 00000100:00000040:0.0:1787695810.424566:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446666d80 x1874531139526272/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424570:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695810.424571:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695810.424573:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96c38. 00000100:00000200:0.0:1787695810.424574:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139526272, offset 0 00000400:00000200:0.0:1787695810.424577:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695810.424579:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695810.424581:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695810.424583:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695810.424585:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695810.424586:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695810.424588:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695810.424590:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16921:16921:256:4294967295] 192.168.201.140@tcp LPNI seq info [16920:16920:8:4294967295] 00000400:00000200:0.0:1787695810.424593:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695810.424597:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695810.424600:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695810.424601:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1444fe2500. 00000800:00000200:0.0:1787695810.424604:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695810.424606:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 1 secs later) 00000800:00000200:0.1:1787695810.424609:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695810.424614:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 1 secs later) 00000800:00000100:0.0:1787695810.424617:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695810.424618:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1444fe2500. 00000400:00000200:0.0:1787695810.424620:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695810.424623:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695810.424625:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96c38 00000400:00000010:0.0:1787695810.424626:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96c38. 00000100:00000001:0.0:1787695810.424630:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695810.424633:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446666d80 x1874531139526272/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424638:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695810.424640:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446666d80 x1874531139526272/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695835 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424644:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.424645:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695810.424647:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695810.424649:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:0.0:1787695810.424651:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695810.424652:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298370917 00000100:00000001:0.0:1787695810.424654:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298382136 00000100:00000001:0.0:1787695810.424655:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298370917 00000100:00000001:0.0:1787695810.424656:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298382136 00000100:00000001:0.0:1787695810.424658:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695810.424659:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.424660:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695810.424663:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695810/real 1787695810] req@ffff8b1446666d80 x1874531139526272/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695810.424667:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695810.424668:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695810.424669:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bf20. 00000400:00000200:0.0:1787695810.424670:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96d48 00000400:00000010:0.0:1787695810.424671:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96d48. 00000100:00000001:0.0:1787695810.424673:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695810.424674:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446666d80 x1874531139526272/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695810.424680:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446666d80 x1874531139526272/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424702:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695810.424704:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.424705:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695810.424706:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695810.424710:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446666d80 x1874531139526272/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424714:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695810.424717:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446666d80 x1874531139526272/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424721:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695810.424723:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.424724:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695810.424724:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.424726:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695810.424727:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695810.424728:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695810.424730:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695810.424731:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695810.424733:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446666d80 x1874531139526272/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695810.424741:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446666d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139526272:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695810.424745:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.424747:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695810.424751:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446664e00 x1874531139526016/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424758:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695810.424759:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695810.424767:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450777000. 02000000:00000001:0.0:1787695810.424770:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695810.424772:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695810.424773:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695810.424775:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695810.424777:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446547c00. 02000000:00000001:0.0:1787695810.424780:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695810.424782:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695810.424783:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446664e00. 00000100:00000001:0.0:1787695810.424786:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695810.424787:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.424789:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695810.424794:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446664380 x1874531139526144/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424800:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695810.424801:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695810.424803:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695810.424805:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695810.424806:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695810.424808:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=23 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695810.424810:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695810.424811:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545c00. 02000000:00000001:0.0:1787695810.424814:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695810.424815:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695810.424817:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446664380. 00000100:00000001:0.0:1787695810.424819:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695810.424820:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.424821:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695810.424825:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446666d80 x1874531139526272/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424831:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695810.424832:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695810.424834:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695810.424836:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695810.424837:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695810.424838:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:0.0:1787695810.424845:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695810.424847:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446546800. 02000000:00000001:0.0:1787695810.424851:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695810.424852:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695810.424853:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446666d80. 00000100:00000001:0.0:1787695810.424856:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695810.424857:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.424859:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.424863:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695810.424865:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.424867:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695810.424868:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695810.424870:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695810.424874:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665180 x1874531139526400/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424880:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695810.424882:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695810.424884:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695810.424886:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695810.424888:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695810.424890:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139526400:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695810.424894:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695810.424896:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695810.424898:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695810.424899:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695810.424902:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695810.424905:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695810.424908:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695810.424911:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bf20. 00000400:00000010:0.0:1787695810.424914:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96d48. 00000100:00000200:0.0:1787695810.424917:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139526400, portal 4 00000100:00000001:0.0:1787695810.424918:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695810.424919:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929558400 : -128555779993216 : ffff8b1446665180) 00000100:00000040:0.0:1787695810.424924:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665180 x1874531139526400/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.424936:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695810.424938:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695810.424940:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96c38. 00000100:00000200:0.0:1787695810.424942:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139526400, offset 0 00000400:00000200:0.0:1787695810.424945:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695810.424948:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695810.424950:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695810.424952:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695810.424954:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695810.424956:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695810.424957:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695810.424959:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16922:16922:256:4294967295] 192.168.201.140@tcp LPNI seq info [16921:16921:8:4294967295] 00000400:00000200:0.0:1787695810.424962:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695810.424967:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695810.424969:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695810.424971:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1444fe2500. 00000800:00000200:0.0:1787695810.424974:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695810.424976:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 1 secs later) 00000800:00000200:0.1:1787695810.424979:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695810.424981:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 1 secs later) 00000800:00000100:0.0:1787695810.424983:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695810.424985:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1444fe2500. 00000400:00000200:0.0:1787695810.424987:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695810.424989:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695810.424991:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96c38 00000400:00000010:0.0:1787695810.424993:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96c38. 00000100:00000001:0.0:1787695810.424994:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695810.424997:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446665180 x1874531139526400/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425002:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695810.425004:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665180 x1874531139526400/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695835 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425017:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.425018:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695810.425021:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695810.425023:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:0.0:1787695810.425025:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695810.425026:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298370589 00000100:00000001:0.0:1787695810.425028:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298382136 00000100:00000001:0.0:1787695810.425030:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298370589 00000100:00000001:0.0:1787695810.425031:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298382136 00000100:00000001:0.0:1787695810.425033:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695810.425034:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.425035:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695810.425038:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695810/real 1787695810] req@ffff8b1446665180 x1874531139526400/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695810.425042:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695810.425043:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695810.425044:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bf20. 00000400:00000200:0.0:1787695810.425046:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96d48 00000400:00000010:0.0:1787695810.425047:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96d48. 00000100:00000001:0.0:1787695810.425048:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695810.425050:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665180 x1874531139526400/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695810.425056:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665180 x1874531139526400/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425059:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695810.425060:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.425061:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695810.425062:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695810.425064:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446665180 x1874531139526400/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425072:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695810.425075:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665180 x1874531139526400/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425079:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695810.425080:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.425081:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695810.425082:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.425083:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695810.425084:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695810.425086:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695810.425087:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695810.425089:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695810.425091:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665180 x1874531139526400/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695810.425096:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139526400:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695810.425099:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695810.425100:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446666680 x1874531139526528/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425105:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695810.425105:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695810.425107:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695810.425108:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695810.425109:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695810.425111:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446666680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139526528:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695810.425112:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695810.425114:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695810.425117:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695810.425119:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695810.425120:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695810.425122:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695810.425124:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695810.425126:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bf20. 00000400:00000010:0.0:1787695810.425128:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96d48. 00000100:00000200:0.0:1787695810.425130:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139526528, portal 10 00000100:00000001:0.0:1787695810.425131:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695810.425132:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929563776 : -128555779987840 : ffff8b1446666680) 00000100:00000040:0.0:1787695810.425136:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446666680 x1874531139526528/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425141:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695810.425142:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695810.425143:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96c38. 00000100:00000200:0.0:1787695810.425145:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139526528, offset 0 00000400:00000200:0.0:1787695810.425147:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695810.425150:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695810.425152:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695810.425154:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695810.425155:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695810.425157:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695810.425158:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695810.425160:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16923:16923:256:4294967295] 192.168.201.140@tcp LPNI seq info [16922:16922:8:4294967295] 00000400:00000200:0.0:1787695810.425163:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695810.425167:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695810.425170:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695810.425172:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1444fe2500. 00000800:00000200:0.0:1787695810.425175:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695810.425177:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 1 secs later) 00000800:00000200:0.1:1787695810.425179:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695810.425181:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 1 secs later) 00000800:00000100:0.0:1787695810.425184:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695810.425188:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1444fe2500. 00000400:00000200:0.0:1787695810.425190:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695810.425192:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695810.425194:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96c38 00000400:00000010:0.0:1787695810.425195:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96c38. 00000100:00000001:0.0:1787695810.425197:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695810.425199:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446666680 x1874531139526528/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425204:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695810.425206:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446666680 x1874531139526528/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695835 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425211:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.425212:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695810.425214:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695810.425216:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:0.0:1787695810.425218:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695810.425219:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298370775 00000100:00000001:0.0:1787695810.425221:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298382136 00000100:00000001:0.0:1787695810.425222:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298370775 00000100:00000001:0.0:1787695810.425224:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298382136 00000100:00000001:0.0:1787695810.425225:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695810.425226:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.425227:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695810.425230:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695810/real 1787695810] req@ffff8b1446666680 x1874531139526528/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695810.425234:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695810.425235:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695810.425236:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bf20. 00000400:00000200:0.0:1787695810.425237:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96d48 00000400:00000010:0.0:1787695810.425238:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96d48. 00000100:00000001:0.0:1787695810.425242:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695810.425245:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446666680 x1874531139526528/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695810.425250:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446666680 x1874531139526528/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425254:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695810.425255:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.425256:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695810.425257:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695810.425259:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446666680 x1874531139526528/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425263:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695810.425265:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446666680 x1874531139526528/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425269:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695810.425271:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.425272:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695810.425273:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.425274:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695810.425275:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695810.425277:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695810.425278:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695810.425280:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695810.425282:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446666680 x1874531139526528/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695810.425287:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446666680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139526528:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695810.425290:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695810.425298:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446664000 x1874531139526656/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425305:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695810.425306:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695810.425308:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695810.425310:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695810.425312:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695810.425315:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446664000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139526656:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695810.425319:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695810.425320:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695810.425322:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695810.425324:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695810.425327:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450777000. 02000000:00000001:0.0:1787695810.425329:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695810.425333:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695810.425336:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bf20. 00000400:00000010:0.0:1787695810.425339:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96d48. 00000100:00000200:0.0:1787695810.425342:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139526656, portal 10 00000100:00000001:0.0:1787695810.425345:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695810.425346:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929553920 : -128555779997696 : ffff8b1446664000) 00000100:00000040:0.0:1787695810.425351:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446664000 x1874531139526656/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425355:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695810.425357:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695810.425358:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96c38. 00000100:00000200:0.0:1787695810.425360:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139526656, offset 0 00000400:00000200:0.0:1787695810.425376:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695810.425379:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695810.425384:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695810.425386:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695810.425389:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695810.425391:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695810.425397:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695810.425399:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16924:16924:256:4294967295] 192.168.201.140@tcp LPNI seq info [16923:16923:8:4294967295] 00000400:00000200:0.0:1787695810.425402:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695810.425407:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695810.425410:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695810.425412:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1444fe2500. 00000800:00000200:0.0:1787695810.425415:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695810.425417:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 1 secs later) 00000800:00000200:0.1:1787695810.425420:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695810.425422:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 8, 1 secs later) 00000800:00000100:0.0:1787695810.425424:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695810.425425:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1444fe2500. 00000400:00000200:0.0:1787695810.425428:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695810.425430:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695810.425432:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96c38 00000400:00000010:0.0:1787695810.425433:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96c38. 00000100:00000001:0.0:1787695810.425435:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695810.425438:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446664000 x1874531139526656/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695835 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425443:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695810.425445:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446664000 x1874531139526656/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695835 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425450:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.425451:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695810.425453:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695810.425455:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:0.0:1787695810.425457:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695810.425458:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298370754 00000100:00000001:0.0:1787695810.425460:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298382136 00000100:00000001:0.0:1787695810.425462:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298370754 00000100:00000001:0.0:1787695810.425463:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298382136 00000100:00000001:0.0:1787695810.425468:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695810.425470:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.425471:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695810.425474:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695810/real 1787695810] req@ffff8b1446664000 x1874531139526656/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695810.425479:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695810.425480:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695810.425481:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bf20. 00000400:00000200:0.0:1787695810.425483:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96d48 00000400:00000010:0.0:1787695810.425484:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96d48. 00000100:00000001:0.0:1787695810.425485:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695810.425487:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446664000 x1874531139526656/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695810.425493:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446664000 x1874531139526656/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425496:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695810.425497:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.425498:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695810.425499:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695810.425501:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446664000 x1874531139526656/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425507:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695810.425511:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446664000 x1874531139526656/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425518:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695810.425520:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.425522:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695810.425523:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.425525:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695810.425527:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695810.425536:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695810.425540:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695810.425542:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695810.425547:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446664000 x1874531139526656/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695810.425556:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446664000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139526656:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695810.425560:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.425562:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695810.425565:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665180 x1874531139526400/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425572:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695810.425573:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695810.425576:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695810.425578:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695810.425580:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695810.425582:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:0.0:1787695810.425584:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695810.425586:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545800. 02000000:00000001:0.0:1787695810.425590:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695810.425591:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695810.425592:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665180. 00000100:00000001:0.0:1787695810.425595:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695810.425596:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.425598:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695810.425602:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446666680 x1874531139526528/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425609:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695810.425611:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695810.425613:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695810.425615:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695810.425616:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695810.425623:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695810.425625:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695810.425627:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545400. 02000000:00000001:0.0:1787695810.425630:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695810.425631:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695810.425633:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446666680. 00000100:00000001:0.0:1787695810.425635:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695810.425637:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.425639:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695810.425643:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446664000 x1874531139526656/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695835 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695810.425650:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695810.425652:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695810.425653:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450777000. 02000000:00000001:0.0:1787695810.425655:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695810.425657:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695810.425658:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=24 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695810.425660:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695810.425662:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446547400. 02000000:00000001:0.0:1787695810.425664:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695810.425666:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695810.425667:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446664000. 00000100:00000001:0.0:1787695810.425669:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695810.425671:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.425673:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695810.425677:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695810.425678:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695810.425680:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695810.425682:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695810.486461:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695810.486468:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695810.486470:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695810.486475:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695810.486478:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695810.486480:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695810.486482:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695810.486484:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695810.486486:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695810.934508:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695810.934524:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695810.934526:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695810.934532:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695810.934534:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695810.934541:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695810.934545:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695811.126534:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695811.126542:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695811.126547:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695811.126548:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695811.126551:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695811.126553:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695811.126556:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695811.126580:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695811.126583:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695811.126584:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695811.126587:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695811.126588:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695811.126660:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695811.126665:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695811.126668:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695811.510424:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695811.510431:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695811.510434:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695811.510440:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695811.510443:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695811.510447:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695811.510450:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695811.510452:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695811.510455:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695811.958539:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695811.958555:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695811.958558:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695811.958563:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695811.958565:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695811.958571:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695811.958575:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695812.150504:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695812.150512:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695812.150518:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695812.150519:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695812.150522:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695812.150523:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695812.150526:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695812.150527:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695812.150529:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695812.150533:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695812.150536:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695812.150542:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695812.150544:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695812.150546:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695812.150549:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695812.150552:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695812.150553:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695812.150555:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695812.150558:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695812.150559:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695812.150617:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695812.150621:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695812.150625:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695812.534459:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695812.534466:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695812.534467:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695812.534471:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695812.534474:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695812.534477:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695812.534479:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695812.534481:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695812.534483:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695812.982506:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695812.982525:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695812.982527:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695812.982533:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695812.982571:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695812.982577:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695812.982581:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695813.174463:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695813.174471:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695813.174473:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695813.174476:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695813.174480:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695813.174482:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695813.174483:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695813.174485:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695813.174488:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695813.174489:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695813.174490:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695813.174491:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695813.174492:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695813.174510:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695813.174511:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695813.174512:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695813.174516:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695813.174517:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695813.174519:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695813.174520:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695813.174522:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695813.174523:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695813.174525:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695813.174526:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695813.174575:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695813.174578:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695813.174581:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695813.558468:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695813.558477:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695813.558479:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695813.558485:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695813.558488:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695813.558491:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695813.558493:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695813.558496:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695813.558498:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695814.006504:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695814.006517:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695814.006519:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695814.006523:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695814.006525:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695814.006530:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695814.006533:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695814.198486:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695814.198493:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695814.198499:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695814.198501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695814.198505:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695814.198506:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695814.198509:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695814.198510:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695814.198513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695814.198515:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695814.198518:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695814.198520:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695814.198618:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695814.198637:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695814.198643:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695814.582436:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695814.582445:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695814.582448:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695814.582452:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695814.582456:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695814.582475:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695814.582478:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695814.582480:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695814.582483:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695815.030499:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695815.030512:0:2405:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000400:3.0:1787695815.030534:0:2405:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695730/real 1787695730] req@ffff8b14466fd500 x1874531139513728/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695815 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787695815.030560:0:2405:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787695815.030562:0:2405:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1787695815.030568:0:2405:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752be18. 00000400:00000200:3.0:1787695815.030577:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8990 00000100:00000001:2.0:1787695815.030581:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000400:00000010:3.0:1787695815.030582:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8990. 00000100:00000001:2.0:1787695815.030589:0:2406:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787695815.030591:0:2405:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000400:2.0:1787695815.030600:0:2406:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695730/real 1787695730] req@ffff8b14466fea00 x1874531139514112/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695815 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1787695815.030601:0:2405:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466fd500 x1874531139513728/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695815 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695815.030638:0:2406:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000200:3.0:1787695815.030639:0:2405:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466fd500 x1874531139513728/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695815 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695815.030641:0:2406:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695815.030645:0:2406:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14456d16e0. 00000400:00000200:2.0:1787695815.030650:0:2406:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b145078a908 00000400:00000010:2.0:1787695815.030655:0:2406:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b145078a908. 00000100:00000001:3.0:1787695815.030657:0:2405:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695815.030659:0:2406:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787695815.030663:0:2405:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1787695815.030666:0:2406:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466fea00 x1874531139514112/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695815 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695815.030676:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695815.030679:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1787695815.030684:0:2406:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466fea00 x1874531139514112/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695815 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695815.030688:0:2405:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b14466fd500 x1874531139513728/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695815 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695815.030710:0:2405:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:3.0:1787695815.030713:0:2405:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b1458081800 already not connected (conn 21, was 15): DISCONN 00000100:00000001:2.0:1787695815.030717:0:2406:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787695815.030717:0:2405:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:3.0:1787695815.030722:0:2405:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695815.030723:0:2406:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695815.030727:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695815.030728:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695815.030733:0:2406:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b14466fea00 x1874531139514112/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695815 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695815.030739:0:2406:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:2.0:1787695815.030741:0:2406:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b144515c000 already not connected (conn 17, was 11): DISCONN 00000100:00000001:3.0:1787695815.030741:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695815.030746:0:2406:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:2.0:1787695815.030748:0:2406:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695815.030748:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695815.030754:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695815.030757:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695815.030766:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695815.030774:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695815.030778:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695815.030778:0:2405:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695815.030782:0:2405:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695815.030783:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695815.030787:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1787695815.030792:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466fd500 x1874531139513728/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695815 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695815.030793:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695815.030797:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695815.030800:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695815.030809:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466fea00 x1874531139514112/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695815 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695815.030809:0:2405:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:3.0:1787695815.030815:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695815.030819:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695815.030822:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695815.030824:0:2406:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:2.0:1787695815.030827:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695815.030829:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466fd500 x1874531139513728/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695815 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695815.030830:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695815.030833:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695815.030840:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466fea00 x1874531139514112/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695815 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695815.030846:0:2405:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466fd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2405:1874531139513728:192.168.201.140@tcp:400:kworker.0 00000100:00100000:2.0:1787695815.030856:0:2406:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466fea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:608c526c-eb6e-4a54-80a5-8969828241eb:2406:1874531139514112:192.168.201.140@tcp:400:kworker.0 00000100:00000001:3.0:1787695815.030856:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695815.030862:0:2405:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787695815.030864:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695815.030867:0:2406:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695815.030868:0:2405:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466fd500 x1874531139513728/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695815 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695815.030875:0:2406:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466fea00 x1874531139514112/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695815 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695815.030877:0:2405:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787695815.030882:0:2405:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1787695815.030887:0:2405:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1448a6a600. 00000100:00000001:2.0:1787695815.030888:0:2406:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695815.030893:0:2406:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787695815.030896:0:2405:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000010:2.0:1787695815.030897:0:2406:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b14618b2800. 00000020:00000001:3.0:1787695815.030900:0:2405:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1787695815.030903:0:2405:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=22 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:2.0:1787695815.030906:0:2406:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787695815.030906:0:2405:0:(genops.c:1230:class_import_put()) Process leaving 00000020:00000001:2.0:1787695815.030910:0:2406:0:(genops.c:1217:class_import_put()) Process entered 02000000:00000010:3.0:1787695815.030910:0:2405:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b14517d5d00. 00000020:00000040:2.0:1787695815.030915:0:2406:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=23 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:3.0:1787695815.030915:0:2405:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787695815.030918:0:2405:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1787695815.030921:0:2405:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466fd500. 00000020:00000001:2.0:1787695815.030922:0:2406:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:3.0:1787695815.030925:0:2405:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 02000000:00000010:2.0:1787695815.030926:0:2406:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b14517d5400. 00000100:00000001:3.0:1787695815.030929:0:2405:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787695815.030931:0:2406:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1787695815.030932:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695815.030934:0:2406:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695815.030938:0:2406:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466fea00. 00000100:00000001:2.0:1787695815.030941:0:2406:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695815.030948:0:2406:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695815.030951:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695815.222467:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695815.222476:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695815.222481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695815.222482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695815.222486:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695815.222487:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695815.222491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695815.222491:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695815.222494:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695815.222495:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695815.222498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695815.222499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695815.222558:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695815.222562:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695815.222588:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787695815.542429:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695815.542441:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787695815.542453:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695815.542457:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695815.542461:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695815.542461:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695815.542471:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3703 00000100:00000001:0.0:1787695815.542476:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695815.542480:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787695815.542482:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695815.542484:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695815.542488:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1787695815.542492:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:0.0:1787695815.542494:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695815.542499:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787695815.542500:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695815.542507:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.542512:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695815.542516:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.542519:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695815.542523:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695815.542526:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:3.0:1787695815.542528:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695815.542532:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787695815.542535:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787695815.542537:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787695815.542541:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.542543:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695815.542545:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.542547:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695815.542550:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787695815.542554:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695815.542557:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695815.542560:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695815.542563:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695815.542565:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695815.542567:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787695815.542569:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695815.542574:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7480. 00000020:00000040:3.0:1787695815.542577:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787695815.542586:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695815.542587:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695815.542590:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695815.542592:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695815.542595:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695815.542602:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b145101dc00. 00000100:00000001:3.0:1787695815.542621:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695815.542628:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7480 x1874531139526784/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695815.542636:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695815.542639:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695815.542643:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043daeda0] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7480 x1874531139526784/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695815.542658:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695815.542660:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695815.542663:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695815.542664:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695815.542665:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695815.542667:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695815.542669:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3703 00000400:00000200:3.0:1787695815.542672:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695815.542674:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695815.542676:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695815.542679:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.542680:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695815.542682:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.542684:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695815.542686:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695815.542687:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:3.0:1787695815.542688:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695815.542690:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000100:00000001:0.0:1787695815.542691:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695815.542695:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695815.542700:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1787695815.542700:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:3.0:1787695815.542703:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:3.0:1787695815.542707:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.542710:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695815.542713:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7480 x1874531139526784/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695815.542713:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.542717:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695815.542721:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:3.0:1787695815.542726:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695815.542730:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695815.542734:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787695815.542739:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787695815.542740:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695815.542743:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695815.542745:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695815.542749:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787695815.542750:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787695815.542755:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1787695815.542756:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787695815.542763:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787695815.542764:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695815.542769:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695815.542774:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139526784:192.168.201.140@tcp:250:kworker.0 00000100:00000010:3.0:1787695815.542776:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5c00. 00000020:00000040:3.0:1787695815.542779:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000100:00000001:0.0:1787695815.542784:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787695815.542785:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695815.542786:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695815.542789:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695815.542790:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787695815.542792:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787695815.542792:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695815.542796:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695815.542799:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 02000000:00000001:0.0:1787695815.542800:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787695815.542805:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695815.542808:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695815.542812:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695815.542814:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5c00 x1874531139526912/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1787695815.542820:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695815.542826:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bf20. 00000100:00000001:3.0:1787695815.542832:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000010:0.0:1787695815.542834:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96d48. 00000100:00000001:3.0:1787695815.542836:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695815.542842:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139526784, portal 25 00000100:00000040:3.0:1787695815.542843:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005bfbcee1] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5c00 x1874531139526912/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695815.542847:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695815.542849:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930157184 : -128555779394432 : ffff8b14466f7480) 00000100:00000040:0.0:1787695815.542858:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7480 x1874531139526784/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695840 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695815.542871:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787695815.542876:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695815.542881:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1787695815.542884:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00080000:3.0:1787695815.542886:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000010:0.0:1787695815.542888:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96c38. 00000100:00000001:3.0:1787695815.542889:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695815.542892:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:0.0:1787695815.542895:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139526784, offset 0 00000100:00000001:3.0:1787695815.542896:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695815.542901:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3703 00000400:00000200:0.0:1787695815.542906:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:3.0:1787695815.542907:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695815.542913:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695815.542914:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695815.542917:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.542918:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1787695815.542922:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695815.542923:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.542926:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695815.542929:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695815.542930:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:3.0:1787695815.542931:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695815.542933:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000400:00000200:0.0:1787695815.542936:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000020:00000001:3.0:1787695815.542936:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:3.0:1787695815.542940:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:3.0:1787695815.542945:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.542946:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1787695815.542948:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1787695815.542949:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.542953:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695815.542954:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000400:00000200:0.0:1787695815.542958:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00080000:3.0:1787695815.542958:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695815.542963:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695815.542966:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695815.542971:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 02000000:00000001:3.0:1787695815.542971:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1787695815.542975:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16925:16925:256:4294967295] 192.168.201.140@tcp LPNI seq info [16924:16924:8:4294967295] 02000000:00000001:3.0:1787695815.542976:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695815.542981:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695815.542985:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695815.542987:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1787695815.542988:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000008:00000001:3.0:1787695815.542991:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695815.542994:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695815.542997:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7800. 00000020:00000040:3.0:1787695815.542999:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=23 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695815.543003:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695815.543005:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695815.543009:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:0.0:1787695815.543010:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 02000000:00000001:3.0:1787695815.543011:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695815.543014:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695815.543019:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675400. 00000100:00000001:3.0:1787695815.543023:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1787695815.543024:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00100000:3.0:1787695815.543030:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7800 x1874531139527040/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:0.0:1787695815.543034:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1444fe2500. 00000100:00000001:3.0:1787695815.543038:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695815.543041:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695815.543044:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7c57981] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7800 x1874531139527040/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1787695815.543048:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695815.543057:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787695815.543058:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695815.543061:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695815.543063:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787695815.543064:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695815.543065:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695815.543067:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695815.543070:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3703 00000400:00000200:3.0:1787695815.543073:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695815.543075:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695815.543077:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787695815.543079:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695815.543086:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695815.543088:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695815.543089:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695815.543092:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298387256 00000100:00000040:3.0:1787695815.543096:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695815.543100:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298382136 00000100:00000001:3.0:1787695815.543101:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695815.543106:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298387256 00000020:00000001:3.0:1787695815.543107:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787695815.543110:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298382136 00000020:00000040:3.0:1787695815.543111:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000100:00000001:0.0:1787695815.543117:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1787695815.543117:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787695815.543122:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695815.543123:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000100:00000001:0.0:1787695815.543125:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695815.543129:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:0.0:1787695815.543130:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787695815.543135:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:0.0:1787695815.543136:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695815.543139:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787695815.543143:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1787695815.543144:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695815.543145:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695815.543149:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787695815.543150:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695815.543155:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000040:0.0:1787695815.543159:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5c00 x1874531139526912/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695815.543159:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695815.543162:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:3.0:1787695815.543165:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695815.543166:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695815.543169:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695815.543172:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695815.543177:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695815.543181:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695815.543186:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787695815.543187:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695815.543191:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787695815.543193:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695815.543197:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787695815.543198:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695815.543202:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695815.543206:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695815.543206:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4e00. 00000100:00100000:0.0:1787695815.543212:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139526912:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695815.543216:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695815.543217:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695815.543219:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695815.543220:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:3.0:1787695815.543223:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000010:0.0:1787695815.543224:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695815.543227:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695815.543233:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787695815.543234:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1787695815.543235:0:2392:0:(lib-socket.c:438:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.201.140:988 00000400:00000010:0.0:1787695815.543237:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b738. 02000000:00000001:3.0:1787695815.543237:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695815.543245:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:0.0:1787695815.543248:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f962a8. 02000000:00000001:3.0:1787695815.543248:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695815.543252:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695815.543253:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139526912, portal 10 00000100:00000001:0.0:1787695815.543256:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000010:3.0:1787695815.543257:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674800. 00000100:00000001:0.0:1787695815.543258:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150912 : -128555779400704 : ffff8b14466f5c00) 00000100:00000040:0.0:1787695815.543267:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5c00 x1874531139526912/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695840 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695815.543268:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695815.543276:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1787695815.543277:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4e00 x1874531139527168/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1787695815.543282:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695815.543286:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96bb0. 00000100:00000200:0.0:1787695815.543291:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139526912, offset 0 00000400:00000200:0.0:1787695815.543302:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695815.543306:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1787695815.543310:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695815.543313:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695815.543317:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695815.543323:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1787695815.543323:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcd77cd6] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4e00 x1874531139527168/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695815.543330:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695815.543336:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695815.543339:0:2392:0:(lib-socket.c:438:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.201.140:988 00000400:00000200:0.0:1787695815.543343:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.543351:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695815.543358:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1787695815.543361:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16926:16926:255:4294967295] 192.168.201.140@tcp LPNI seq info [16925:16925:7:4294967295] 00000100:00080000:3.0:1787695815.543386:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1787695815.543387:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695815.543389:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695815.543391:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695815.543397:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3703 00000400:00000200:3.0:1787695815.543401:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695815.543404:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695815.543406:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695815.543409:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.543411:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695815.543412:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.543414:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695815.543415:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695815.543416:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:3.0:1787695815.543417:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695815.543419:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:3.0:1787695815.543420:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:3.0:1787695815.543421:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:3.0:1787695815.543423:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.543429:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1787695815.543433:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695815.543439:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.543442:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695815.543448:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:3.0:1787695815.543453:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:0.0:1787695815.543454:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:3.0:1787695815.543457:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695815.543461:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:0.0:1787695815.543463:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695815.543464:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695815.543468:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1787695815.543469:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1444fe2600. 00000100:00000001:3.0:1787695815.543470:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695815.543475:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000800:00000200:0.0:1787695815.543479:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000008:00000001:3.0:1787695815.543479:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695815.543483:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1787695815.543487:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000010:3.0:1787695815.543489:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5880. 00000100:00000001:0.0:1787695815.543492:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695815.543493:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 00000100:00000040:0.0:1787695815.543497:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 02000000:00000001:3.0:1787695815.543498:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695815.543501:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695815.543506:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695815.543506:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787695815.543510:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298377158 02000000:00000001:3.0:1787695815.543511:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695815.543514:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695815.543515:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298387256 02000000:00000010:3.0:1787695815.543518:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 00000100:00000001:0.0:1787695815.543519:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298377158 00000100:00000001:0.0:1787695815.543524:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298387256 00000100:00000001:3.0:1787695815.543524:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695815.543526:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695815.543529:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695815.543531:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5880 x1874531139527296/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695815.543532:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787695815.543540:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695815.543542:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695815.543549:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000827e7dbe] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5880 x1874531139527296/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695815.543551:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7800 x1874531139527040/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695815.543579:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695815.543579:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695815.543582:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695815.543586:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1787695815.543587:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1787695815.543588:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695815.543592:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695815.543597:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1787695815.543598:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139527040:192.168.201.140@tcp:38:kworker.0 00000100:00000001:3.0:1787695815.543601:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787695815.543614:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695815.543617:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695815.543618:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695815.543623:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1787695815.543623:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000010:0.0:1787695815.543628:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450777000. 00000100:00000001:3.0:1787695815.543629:0:38:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:0.0:1787695815.543635:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695815.543635:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3703 00000400:00000200:3.0:1787695815.543642:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695815.543650:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695815.543651:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000010:0.0:1787695815.543654:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bf78. 00000100:00080000:3.0:1787695815.543656:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000400:00000010:0.0:1787695815.543660:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000040:3.0:1787695815.543663:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000200:0.0:1787695815.543665:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139527040, portal 10 00000100:00000001:0.0:1787695815.543669:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787695815.543669:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695815.543672:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930158080 : -128555779393536 : ffff8b14466f7800) 00000100:00000040:3.0:1787695815.543676:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.543681:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000040:0.0:1787695815.543682:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7800 x1874531139527040/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695840 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695815.543686:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695815.543690:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000100:00000001:0.0:1787695815.543694:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1787695815.543694:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000040:0.0:1787695815.543699:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000020:00000040:3.0:1787695815.543700:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000400:00000010:0.0:1787695815.543701:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f967f8. 00000020:00000001:3.0:1787695815.543705:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000200:0.0:1787695815.543706:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139527040, offset 0 00000020:00000001:3.0:1787695815.543712:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000200:0.0:1787695815.543714:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695815.543721:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695815.543721:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.543727:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1787695815.543728:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000040:3.0:1787695815.543733:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695815.543734:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1787695815.543740:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:0.0:1787695815.543742:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000020:00000040:3.0:1787695815.543746:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000400:00000200:0.0:1787695815.543748:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00080000:3.0:1787695815.543754:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:0.0:1787695815.543755:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.543762:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695815.543764:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16927:16927:254:4294967295] 192.168.201.140@tcp LPNI seq info [16926:16926:6:4294967295] 02000000:00000001:3.0:1787695815.543765:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695815.543769:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695815.543773:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695815.543777:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695815.543777:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695815.543780:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695815.543783:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695815.543785:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695815.543790:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000010:3.0:1787695815.543795:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6680. 00000800:00000200:0.0:1787695815.543799:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000020:00000040:3.0:1787695815.543800:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:3.0:1787695815.543804:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000010:0.0:1787695815.543805:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1444fe2100. 02000000:00000001:3.0:1787695815.543806:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695815.543811:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695815.543812:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:0.0:1787695815.543816:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:3.0:1787695815.543816:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695815.543823:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 00000800:00000200:0.1:1787695815.543823:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695815.543829:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695815.543829:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695815.543831:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000100:00100000:3.0:1787695815.543836:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6680 x1874531139527424/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1787695815.543837:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695815.543841:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298376006 00000100:00000001:0.0:1787695815.543846:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298387256 00000100:00000001:0.0:1787695815.543851:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298376006 00000100:00000001:3.0:1787695815.543851:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695815.543854:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695815.543857:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f6d2abf9] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6680 x1874531139527424/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695815.543863:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298387256 00000100:00000001:0.0:1787695815.543868:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695815.543871:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695815.543872:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695815.543876:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695815.543877:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695815.543882:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695815.543882:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695815.543885:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787695815.543887:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1787695815.543887:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695815.543890:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1787695815.543891:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787695815.543894:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787695815.543895:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3703 00000100:00000001:0.0:1787695815.543897:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695815.543900:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000400:00000200:3.0:1787695815.543901:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695815.543907:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000040:0.0:1787695815.543908:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4e00 x1874531139527168/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695815.543909:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695815.543914:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.543917:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695815.543920:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.543925:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695815.543928:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787695815.543933:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1787695815.543936:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000100:00000001:0.0:1787695815.543938:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695815.543939:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695815.543943:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 02000000:00000001:0.0:1787695815.543944:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1787695815.543945:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000020:00000001:3.0:1787695815.543949:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 02000000:00000001:0.0:1787695815.543950:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1787695815.543952:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695815.543954:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695815.543955:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:0.0:1787695815.543960:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139527168:192.168.201.140@tcp:8:kworker.0 00000100:00000040:3.0:1787695815.543960:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695815.543965:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695815.543968:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1787695815.543969:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 02000000:00000001:0.0:1787695815.543971:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1787695815.543973:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000001:0.0:1787695815.543974:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695815.543977:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695815.543978:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787695815.543980:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695815.543983:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695815.543985:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695815.543986:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5c00. 00000100:00000001:3.0:1787695815.543986:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695815.543989:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787695815.543992:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787695815.543992:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695815.543998:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695815.544001:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7100. 00000400:00000200:0.0:1787695815.544003:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000040:3.0:1787695815.544004:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=24 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:3.0:1787695815.544009:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695815.544010:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695815.544011:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617be18. 00000100:00000001:3.0:1787695815.544012:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695815.544014:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695815.544017:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695815.544020:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96440. 02000000:00000010:3.0:1787695815.544021:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 00000100:00000200:0.0:1787695815.544028:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139527168, portal 4 00000100:00000001:3.0:1787695815.544028:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695815.544033:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695815.544035:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930147328 : -128555779404288 : ffff8b14466f4e00) 00000100:00100000:3.0:1787695815.544039:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7100 x1874531139527552/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1787695815.544045:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4e00 x1874531139527168/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695840 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695815.544063:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787695815.544063:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695815.544068:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695815.544072:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695815.544075:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96ee0. 00000100:00000200:0.0:1787695815.544079:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139527168, offset 0 00000400:00000200:0.0:1787695815.544085:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695815.544086:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000004e8abd7] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7100 x1874531139527552/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695815.544091:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695815.544097:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695815.544104:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695815.544109:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695815.544113:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1787695815.544114:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695815.544116:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695815.544121:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16928:16928:253:4294967295] 192.168.201.140@tcp LPNI seq info [16927:16927:5:4294967295] 00000100:00000040:3.0:1787695815.544121:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1787695815.544130:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695815.544132:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695815.544140:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1787695815.544141:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000040:3.0:1787695815.544148:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:0.0:1787695815.544153:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695815.544154:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3713) 00000800:00000010:0.0:1787695815.544159:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14619e0900. 00000800:00000200:0.0:1787695815.544165:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695815.544170:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695815.544173:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695815.544177:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:0.0:1787695815.544180:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695815.544182:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298376036 00000100:00000001:0.0:1787695815.544185:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298387256 00000100:00000001:0.0:1787695815.544187:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298376036 00000100:00000001:0.0:1787695815.544190:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298387256 00000100:00000001:0.0:1787695815.544192:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695815.544195:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695815.544197:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695815.544201:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5880 x1874531139527296/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695815.544206:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695815.544208:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695815.544214:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695815.544215:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695815.544217:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695815.544219:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139527296:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695815.544222:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695815.544223:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695815.544224:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695815.544226:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695815.544228:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f7400. 02000000:00000001:0.0:1787695815.544229:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695815.544232:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695815.544234:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b058. 00000400:00000010:0.0:1787695815.544236:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f963b8. 00000100:00000200:0.0:1787695815.544238:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139527296, portal 4 00000100:00000001:0.0:1787695815.544240:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695815.544241:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150016 : -128555779401600 : ffff8b14466f5880) 00000100:00000040:0.0:1787695815.544245:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5880 x1874531139527296/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695840 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695815.544250:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695815.544251:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695815.544253:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96000. 00000100:00000200:0.0:1787695815.544255:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139527296, offset 0 00000400:00000200:0.0:1787695815.544257:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695815.544260:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695815.544262:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695815.544264:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695815.544266:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695815.544268:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695815.544270:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695815.544272:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16929:16929:252:4294967295] 192.168.201.140@tcp LPNI seq info [16928:16928:4:4294967295] 00000400:00000200:0.0:1787695815.544274:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695815.544279:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695815.544286:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695815.544288:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14619e0d00. 00000800:00000200:0.0:1787695815.544291:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695815.544293:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695815.544295:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695815.544297:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:0.0:1787695815.544299:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695815.544301:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298375708 00000100:00000001:0.0:1787695815.544303:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298387256 00000100:00000001:0.0:1787695815.544304:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298375708 00000100:00000001:0.0:1787695815.544306:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298387256 00000100:00000001:0.0:1787695815.544307:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695815.544309:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695815.544310:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695815.544312:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6680 x1874531139527424/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695815.544317:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695815.544317:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695815.544319:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695815.544320:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695815.544321:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695815.544323:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139527424:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695815.544325:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695815.544327:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695815.544328:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695815.544329:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695815.544330:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5400. 02000000:00000001:0.0:1787695815.544331:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695815.544333:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695815.544335:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bb00. 00000400:00000010:0.0:1787695815.544337:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96990. 00000100:00000200:0.0:1787695815.544342:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139527424, portal 10 00000100:00000001:0.0:1787695815.544343:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695815.544344:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930153600 : -128555779398016 : ffff8b14466f6680) 00000100:00000040:0.0:1787695815.544349:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6680 x1874531139527424/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695840 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695815.544353:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695815.544355:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695815.544356:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96660. 00000100:00000200:0.0:1787695815.544357:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139527424, offset 0 00000400:00000200:0.0:1787695815.544360:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695815.544390:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695815.544395:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695815.544397:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695815.544400:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695815.544401:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695815.544404:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695815.544406:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16930:16930:251:4294967295] 192.168.201.140@tcp LPNI seq info [16929:16929:3:4294967295] 00000400:00000200:0.0:1787695815.544409:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695815.544413:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695815.544416:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695815.544418:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14619e0c00. 00000800:00000200:0.0:1787695815.544421:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695815.544423:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695815.544425:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695815.544427:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:0.0:1787695815.544429:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695815.544431:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298375895 00000100:00000001:0.0:1787695815.544432:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298387256 00000100:00000001:0.0:1787695815.544434:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298375895 00000100:00000001:0.0:1787695815.544435:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298387256 00000100:00000001:0.0:1787695815.544436:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695815.544442:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695815.544443:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695815.544445:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695815.544447:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695815.544449:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787695815.544450:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695815.544451:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695815.544452:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695815.544455:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7100 x1874531139527552/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695815.544460:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695815.544461:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695815.544462:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695815.544463:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695815.544465:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695815.544467:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139527552:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695815.544469:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695815.544470:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695815.544471:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695815.544472:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695815.544474:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5000. 02000000:00000001:0.0:1787695815.544476:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695815.544478:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695815.544480:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b580. 00000400:00000010:0.0:1787695815.544482:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 00000100:00000200:0.0:1787695815.544484:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139527552, portal 10 00000100:00000001:0.0:1787695815.544486:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695815.544486:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930156288 : -128555779395328 : ffff8b14466f7100) 00000100:00000040:0.0:1787695815.544490:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7100 x1874531139527552/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695840 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695815.544494:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695815.544496:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695815.544497:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96220. 00000100:00000200:0.0:1787695815.544499:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139527552, offset 0 00000400:00000200:0.0:1787695815.544506:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695815.544509:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695815.544511:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695815.544513:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695815.544514:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695815.544516:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695815.544517:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695815.544520:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16931:16931:250:4294967295] 192.168.201.140@tcp LPNI seq info [16930:16930:2:4294967295] 00000400:00000200:0.0:1787695815.544522:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695815.544526:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695815.544529:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695815.544531:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981300. 00000800:00000200:0.0:1787695815.544534:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695815.544536:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695815.544538:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695815.544540:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:0.0:1787695815.544543:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695815.544544:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298375873 00000100:00000001:0.0:1787695815.544545:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298387256 00000100:00000001:0.0:1787695815.544547:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298375873 00000100:00000001:0.0:1787695815.544548:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298387256 00000100:00000001:0.0:1787695815.544549:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695815.544550:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695815.544552:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695815.544553:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695815.544555:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695815.544556:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787695815.544557:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695815.544558:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695815.544559:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695815.544561:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695815.606463:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695815.606470:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695815.606472:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695815.606476:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695815.606478:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695815.606481:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695815.606483:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695815.606485:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695815.606487:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695816.054460:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695816.054460:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695816.054474:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695816.054475:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695816.054478:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695816.054479:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695816.054485:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695816.054486:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695816.054488:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695816.054489:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695816.054501:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695816.054501:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695816.054507:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695816.054507:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695816.246475:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695816.246481:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695816.246486:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695816.246500:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695816.246503:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695816.246506:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695816.246508:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695816.246513:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695816.246514:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695816.246516:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695816.246518:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695816.246520:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695816.246521:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695816.246523:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695816.246526:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695816.246527:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695816.246528:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695816.246529:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695816.246529:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695816.246530:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695816.246531:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695816.246532:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695816.246533:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695816.246534:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695816.246537:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695816.246537:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695816.246540:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695816.246541:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695816.246541:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695816.246543:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695816.246543:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695816.246544:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695816.246545:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695816.246546:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695816.246547:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695816.246548:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695816.246648:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695816.246659:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695816.246662:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787695816.374684:0:2392:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1021 -> 192.168.201.140:988 00000400:00000100:2.0:1787695816.374742:0:2392:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.140@tcp at host 192.168.201.140 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1787695816.374752:0:2392:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695816.374756:0:2392:0:(router.c:1791:lnet_notify()) 192.168.201.40@tcp notifying 192.168.201.140@tcp: down 00000400:00000200:2.0:1787695816.374761:0:2392:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695816.374765:0:2392:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:2.0:1787695816.374770:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:2.0:1787695816.374775:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1444fe2500. 00000400:00000200:2.0:1787695816.374781:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1787695816.374786:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1787695816.374792:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96c38 00000400:00000010:2.0:1787695816.374795:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96c38. 00000100:00000001:2.0:1787695816.374800:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695816.374808:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f7480 x1874531139526784/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695840 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695816.374836:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695816.374840:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7480 x1874531139526784/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695840 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695816.374849:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695816.374851:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787695816.374860:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:2.0:1787695816.374863:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1444fe2600. 00000400:00000200:2.0:1787695816.374881:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1787695816.374887:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000100:00000001:0.0:1787695816.374889:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1787695816.374891:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96bb0 00000100:00000001:0.0:1787695816.374895:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000010:2.0:1787695816.374896:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96bb0. 00000100:00000001:2.0:1787695816.374902:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787695816.374904:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000200:2.0:1787695816.374909:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f5c00 x1874531139526912/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695840 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695816.374919:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695815/real 1787695816] req@ffff8b14466f7480 x1874531139526784/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695816.374922:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695816.374930:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5c00 x1874531139526912/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695840 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695816.374940:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1787695816.374941:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787695816.374944:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787695816.374945:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000010:0.0:1787695816.374949:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bf20. 00000800:00000100:2.0:1787695816.374954:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000400:00000200:0.0:1787695816.374958:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96d48 00000800:00000010:2.0:1787695816.374960:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1444fe2100. 00000400:00000010:0.0:1787695816.374964:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96d48. 00000400:00000200:2.0:1787695816.374966:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:0.0:1787695816.374970:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:2.0:1787695816.374973:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1787695816.374979:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f967f8 00000100:00000200:0.0:1787695816.374980:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7480 x1874531139526784/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1787695816.374983:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f967f8. 00000100:00000001:2.0:1787695816.374989:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695816.374998:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f7800 x1874531139527040/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695840 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695816.375002:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7480 x1874531139526784/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695816.375019:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787695816.375024:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:2.0:1787695816.375027:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7800 x1874531139527040/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695840 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.375028:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.375035:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695816.375039:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695816.375043:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695816.375049:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7480 x1874531139526784/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.375057:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695816.375063:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1787695816.375064:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7480 x1874531139526784/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1787695816.375073:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:2.0:1787695816.375081:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14619e0900. 00000100:00000001:0.0:1787695816.375082:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:2.0:1787695816.375087:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1787695816.375092:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1787695816.375095:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96ee0 00000100:00000001:0.0:1787695816.375101:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1787695816.375103:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96ee0. 00000100:00000001:0.0:1787695816.375105:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695816.375109:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787695816.375110:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.375115:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000200:2.0:1787695816.375117:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f4e00 x1874531139527168/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695840 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.375127:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695816.375130:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1787695816.375134:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:2.0:1787695816.375135:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4e00 x1874531139527168/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695840 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787695816.375143:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695816.375149:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695816.375150:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1787695816.375153:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1787695816.375160:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7480 x1874531139526784/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1787695816.375165:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:2.0:1787695816.375172:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14619e0d00. 00000400:00000200:2.0:1787695816.375178:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00100000:0.0:1787695816.375181:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139526784:192.168.201.140@tcp:250:kworker.0 00000400:00000200:2.0:1787695816.375189:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000100:00000001:0.0:1787695816.375191:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695816.375194:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96000 00000100:00000001:0.0:1787695816.375195:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000400:00000010:2.0:1787695816.375200:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96000. 00000100:00000040:0.0:1787695816.375203:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7480 x1874531139526784/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695816.375205:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695816.375214:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f5880 x1874531139527296/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695840 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.375216:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695816.375221:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695816.375227:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450774400. 00000100:00000001:2.0:1787695816.375228:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695816.375235:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5880 x1874531139527296/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695840 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787695816.375236:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695816.375241:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695816.375245:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000100:00000001:2.0:1787695816.375246:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695816.375250:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:2.0:1787695816.375251:0:2392:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000010:0.0:1787695816.375254:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b145101dc00. 00000800:00000100:2.0:1787695816.375261:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 02000000:00000001:0.0:1787695816.375262:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000800:00000010:2.0:1787695816.375264:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14619e0c00. 02000000:00000001:0.0:1787695816.375266:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695816.375271:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7480. 00000400:00000200:2.0:1787695816.375271:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:0.0:1787695816.375278:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000400:00000200:2.0:1787695816.375280:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000100:00000001:0.0:1787695816.375281:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.375286:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695816.375286:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96660 00000100:00000001:0.0:1787695816.375291:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:2.0:1787695816.375291:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96660. 00000100:00000001:0.0:1787695816.375295:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695816.375298:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787695816.375299:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000200:2.0:1787695816.375306:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f6680 x1874531139527424/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695840 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695816.375308:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695815/real 1787695816] req@ffff8b14466f4e00 x1874531139527168/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695816.375333:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695816.375343:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787695816.375346:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000400:00000010:0.0:1787695816.375347:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617be18. 00000400:00000200:0.0:1787695816.375353:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96440 00000100:00000040:2.0:1787695816.375354:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6680 x1874531139527424/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695840 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1787695816.375357:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96440. 00000100:00000001:0.0:1787695816.375400:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695816.375408:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4e00 x1874531139527168/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695816.375411:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695816.375415:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000200:0.0:1787695816.375426:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4e00 x1874531139527168/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1787695816.375429:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:2.0:1787695816.375436:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981300. 00000400:00000200:2.0:1787695816.375441:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:0.0:1787695816.375444:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695816.375448:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1787695816.375449:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000100:00000001:0.0:1787695816.375454:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:2.0:1787695816.375456:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96220 00000100:00000001:0.0:1787695816.375459:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1787695816.375460:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96220. 00000100:00000001:2.0:1787695816.375466:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:0.0:1787695816.375471:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4e00 x1874531139527168/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695816.375476:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f7100 x1874531139527552/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695840 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.375499:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695816.375502:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695816.375509:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4e00 x1874531139527168/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695816.375510:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7100 x1874531139527552/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695840 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695816.375542:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695816.375544:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695816.375546:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695816.375550:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.375554:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695816.375555:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.375557:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695816.375559:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695816.375562:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695816.375567:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695816.375570:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695816.375575:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4e00 x1874531139527168/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695816.375615:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139527168:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695816.375620:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695816.375624:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695815/real 1787695816] req@ffff8b14466f5880 x1874531139527296/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695816.375638:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695816.375640:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695816.375642:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b058. 00000400:00000200:0.0:1787695816.375646:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f963b8 00000400:00000010:0.0:1787695816.375648:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f963b8. 00000100:00000001:0.0:1787695816.375651:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695816.375654:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5880 x1874531139527296/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695816.375662:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5880 x1874531139527296/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.375669:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695816.375670:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.375673:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695816.375674:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695816.375679:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5880 x1874531139527296/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.375686:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695816.375690:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5880 x1874531139527296/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.375697:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695816.375701:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.375703:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695816.375704:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.375713:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695816.375715:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695816.375717:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695816.375722:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695816.375724:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695816.375729:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5880 x1874531139527296/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695816.375738:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139527296:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695816.375743:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695816.375747:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695815/real 1787695816] req@ffff8b14466f6680 x1874531139527424/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695816.375756:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695816.375758:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695816.375760:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bb00. 00000400:00000200:0.0:1787695816.375763:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96990 00000400:00000010:0.0:1787695816.375766:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96990. 00000100:00000001:0.0:1787695816.375768:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695816.375772:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6680 x1874531139527424/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695816.375782:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6680 x1874531139527424/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.375789:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695816.375791:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.375793:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695816.375795:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695816.375800:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6680 x1874531139527424/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.375808:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695816.375818:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6680 x1874531139527424/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.375826:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695816.375828:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.375829:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695816.375830:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.375832:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695816.375833:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695816.375836:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695816.375840:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695816.375842:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695816.375845:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6680 x1874531139527424/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695816.375852:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139527424:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695816.375857:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695816.375860:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695815/real 1787695816] req@ffff8b14466f5c00 x1874531139526912/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695816.375869:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695816.375870:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695816.375872:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b738. 00000400:00000200:0.0:1787695816.375876:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f962a8 00000400:00000010:0.0:1787695816.375877:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f962a8. 00000100:00000001:0.0:1787695816.375880:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695816.375884:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5c00 x1874531139526912/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695816.375892:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5c00 x1874531139526912/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.375898:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695816.375906:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.375908:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695816.375909:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695816.375912:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5c00 x1874531139526912/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.375918:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695816.375922:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5c00 x1874531139526912/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.375928:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695816.375930:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.375938:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695816.375939:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.375941:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695816.375943:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695816.375945:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695816.375949:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695816.375951:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695816.375955:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5c00 x1874531139526912/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695816.375962:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139526912:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695816.375967:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695816.375970:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695815/real 1787695816] req@ffff8b14466f7800 x1874531139527040/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695816.375978:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695816.375979:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695816.375981:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bf78. 00000400:00000200:0.0:1787695816.375984:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695816.375985:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695816.375994:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695816.375998:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7800 x1874531139527040/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695816.376006:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7800 x1874531139527040/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.376013:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695816.376014:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.376016:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695816.376018:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695816.376022:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7800 x1874531139527040/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.376029:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695816.376033:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7800 x1874531139527040/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.376045:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695816.376047:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.376049:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695816.376050:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.376052:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695816.376053:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695816.376056:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695816.376060:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695816.376062:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695816.376066:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7800 x1874531139527040/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695816.376074:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139527040:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695816.376079:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695816.376087:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695816.376092:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4e00 x1874531139527168/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.376099:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695816.376101:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695816.376103:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5c00. 02000000:00000001:0.0:1787695816.376107:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695816.376110:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695816.376111:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:0.0:1787695816.376113:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695816.376115:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674800. 02000000:00000001:0.0:1787695816.376119:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695816.376120:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695816.376122:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4e00. 00000100:00000001:0.0:1787695816.376125:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695816.376126:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.376128:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695816.376132:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5880 x1874531139527296/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.376139:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695816.376140:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695816.376142:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f7400. 02000000:00000001:0.0:1787695816.376145:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695816.376146:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695816.376148:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:0.0:1787695816.376150:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695816.376151:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:0.0:1787695816.376154:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695816.376155:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695816.376156:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5880. 00000100:00000001:0.0:1787695816.376158:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695816.376159:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.376161:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695816.376165:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6680 x1874531139527424/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.376184:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695816.376185:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695816.376187:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5400. 02000000:00000001:0.0:1787695816.376190:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695816.376192:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695816.376193:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695816.376195:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695816.376197:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:0.0:1787695816.376200:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695816.376201:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695816.376202:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6680. 00000100:00000001:0.0:1787695816.376204:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695816.376205:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.376206:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695816.376210:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5c00 x1874531139526912/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.376218:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695816.376219:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695816.376221:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695816.376223:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695816.376224:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695816.376226:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695816.376227:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695816.376229:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:0.0:1787695816.376232:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695816.376233:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695816.376234:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5c00. 00000100:00000001:0.0:1787695816.376237:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695816.376238:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.376239:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695816.376243:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7800 x1874531139527040/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.376249:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695816.376251:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695816.376258:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450777000. 02000000:00000001:0.0:1787695816.376261:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695816.376262:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695816.376263:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=22 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695816.376266:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695816.376267:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675400. 02000000:00000001:0.0:1787695816.376270:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695816.376272:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695816.376273:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7800. 00000100:00000001:0.0:1787695816.376276:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695816.376277:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.376279:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695816.376282:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695816.376283:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695816.376285:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695816.376289:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695815/real 1787695816] req@ffff8b14466f7100 x1874531139527552/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695816.376299:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695816.376300:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695816.376302:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b580. 00000400:00000200:0.0:1787695816.376305:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:0.0:1787695816.376306:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:0.0:1787695816.376309:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695816.376312:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7100 x1874531139527552/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695816.376322:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7100 x1874531139527552/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.376329:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695816.376330:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.376332:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695816.376334:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695816.376337:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7100 x1874531139527552/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.376357:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695816.376362:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7100 x1874531139527552/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.376392:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695816.376394:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.376395:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695816.376397:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.376399:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695816.376400:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695816.376402:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695816.376406:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695816.376407:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695816.376410:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7100 x1874531139527552/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695816.376416:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139527552:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695816.376419:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.376420:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695816.376422:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7100 x1874531139527552/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695840 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695816.376426:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695816.376427:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695816.376428:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5000. 02000000:00000001:0.0:1787695816.376430:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695816.376431:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695816.376432:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=23 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695816.376434:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695816.376435:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:0.0:1787695816.376441:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695816.376442:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695816.376444:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7100. 00000100:00000001:0.0:1787695816.376445:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695816.376446:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.376448:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695816.376453:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695816.376455:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695816.376456:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695816.376457:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695816.566764:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695816.566780:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695816.566783:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695816.566788:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695816.566790:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695816.566795:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695816.566816:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695816.630431:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695816.630437:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695816.630438:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695816.630443:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695816.630445:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695816.630449:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695816.630451:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695816.630453:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695816.630455:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695817.078459:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695817.078475:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695817.078478:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695817.078478:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695817.078486:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695817.078488:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695817.078489:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695817.078492:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695817.078497:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695817.078498:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695817.078507:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695817.078509:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695817.078512:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695817.078515:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695817.270438:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695817.270443:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695817.270447:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695817.270448:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695817.270451:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695817.270452:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695817.270454:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695817.270455:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695817.270457:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695817.270458:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695817.270460:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695817.270461:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695817.270503:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695817.270508:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695817.270511:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787695817.590498:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695817.590517:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695817.590520:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695817.590526:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695817.590529:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695817.590535:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695817.590539:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695817.654416:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695817.654425:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695817.654427:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695817.654432:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695817.654436:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695817.654439:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695817.654441:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695817.654444:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695817.654446:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695818.102500:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695818.102500:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695818.102523:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695818.102525:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695818.102535:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695818.102536:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695818.102550:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695818.102551:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695818.102573:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695818.102573:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695818.102582:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695818.102588:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695818.102591:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695818.102597:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695818.294483:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695818.294491:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695818.294497:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695818.294499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695818.294503:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695818.294504:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695818.294508:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695818.294509:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695818.294513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695818.294514:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695818.294517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695818.294519:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695818.294595:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695818.294599:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695818.294602:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787695818.614473:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695818.614487:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695818.614490:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695818.614494:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695818.614497:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695818.614502:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695818.614505:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695818.678432:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695818.678443:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695818.678446:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695818.678453:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695818.678457:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695818.678461:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695818.678463:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695818.678467:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695818.678470:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695819.126463:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695819.126463:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695819.126477:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695819.126478:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695819.126480:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695819.126481:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695819.126488:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695819.126489:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695819.126491:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695819.126496:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695819.126499:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695819.126508:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695819.126512:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695819.126515:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695819.318444:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695819.318450:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695819.318454:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695819.318455:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695819.318458:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695819.318459:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695819.318461:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695819.318462:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695819.318464:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695819.318465:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695819.318467:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695819.318468:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695819.318522:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695819.318527:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695819.318531:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787695819.638759:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695819.638775:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695819.638778:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695819.638783:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695819.638786:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695819.638793:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695819.638797:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695819.702450:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695819.702459:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695819.702461:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695819.702466:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695819.702469:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695819.702473:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695819.702476:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695819.702479:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695819.702481:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695820.150457:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695820.150457:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695820.150466:0:2406:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787695820.150466:0:2405:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000400:2.0:1787695820.150484:0:2406:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695735/real 1787695735] req@ffff8b14466ff800 x1874531139515392/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:3.0:1787695820.150485:0:2405:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695735/real 1787695735] req@ffff8b14466fdc00 x1874531139515008/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695820.150537:0:2406:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695820.150539:0:2406:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695820.150542:0:2406:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14456d1210. 00000400:00000200:2.0:1787695820.150548:0:2406:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b145078a2a8 00000400:00000010:2.0:1787695820.150551:0:2406:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b145078a2a8. 00000100:00000001:2.0:1787695820.150554:0:2406:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000001:3.0:1787695820.150556:0:2405:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000200:2.0:1787695820.150559:0:2406:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466ff800 x1874531139515392/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787695820.150559:0:2405:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1787695820.150563:0:2405:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752b580. 00000400:00000200:3.0:1787695820.150569:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f82a8 00000100:00000200:2.0:1787695820.150574:0:2406:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466ff800 x1874531139515392/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1787695820.150574:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f82a8. 00000100:00000001:3.0:1787695820.150578:0:2405:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787695820.150585:0:2405:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466fdc00 x1874531139515008/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695820.150586:0:2406:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695820.150588:0:2406:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695820.150592:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695820.150595:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1787695820.150600:0:2405:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466fdc00 x1874531139515008/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695820.150602:0:2406:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b14466ff800 x1874531139515392/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695820.150620:0:2405:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695820.150622:0:2406:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00000001:3.0:1787695820.150623:0:2405:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787695820.150626:0:2406:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b144515c000 already not connected (conn 18, was 11): DISCONN 00000100:00000001:3.0:1787695820.150627:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695820.150630:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695820.150631:0:2406:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:2.0:1787695820.150633:0:2406:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787695820.150636:0:2405:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b14466fdc00 x1874531139515008/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695820.150641:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695820.150642:0:2405:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00000001:2.0:1787695820.150645:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695820.150646:0:2405:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b1458081800 already not connected (conn 22, was 15): DISCONN 00000100:00000001:2.0:1787695820.150666:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695820.150666:0:2405:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:3.0:1787695820.150670:0:2405:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695820.150671:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695820.150675:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695820.150676:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695820.150678:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695820.150680:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695820.150681:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695820.150682:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695820.150685:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1787695820.150689:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466ff800 x1874531139515392/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695820.150690:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695820.150694:0:2405:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695820.150697:0:2405:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695820.150703:0:2406:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:2.0:1787695820.150707:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695820.150709:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:3.0:1787695820.150709:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466fdc00 x1874531139515008/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695820.150711:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695820.150717:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466ff800 x1874531139515392/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695820.150737:0:2406:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466ff800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:608c526c-eb6e-4a54-80a5-8969828241eb:2406:1874531139515392:192.168.201.140@tcp:400:kworker.0 00000100:00000001:3.0:1787695820.150737:0:2405:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:3.0:1787695820.150744:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695820.150745:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695820.150751:0:2406:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787695820.150751:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695820.150755:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695820.150757:0:2406:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466ff800 x1874531139515392/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695820.150762:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466fdc00 x1874531139515008/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695820.150767:0:2406:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695820.150771:0:2406:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:3.0:1787695820.150775:0:2405:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466fdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2405:1874531139515008:192.168.201.140@tcp:400:kworker.0 02000000:00000010:2.0:1787695820.150776:0:2406:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b14532da800. 02000000:00000001:2.0:1787695820.150783:0:2406:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1787695820.150784:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787695820.150788:0:2406:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:3.0:1787695820.150788:0:2405:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000040:2.0:1787695820.150792:0:2406:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=22 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000040:3.0:1787695820.150794:0:2405:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466fdc00 x1874531139515008/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787695820.150796:0:2406:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695820.150800:0:2406:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1461888e00. 00000100:00000001:3.0:1787695820.150801:0:2405:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695820.150804:0:2406:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787695820.150804:0:2405:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787695820.150807:0:2406:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000010:3.0:1787695820.150808:0:2405:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1448a6ba00. 00000100:00000010:2.0:1787695820.150811:0:2406:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466ff800. 02000000:00000001:3.0:1787695820.150814:0:2405:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787695820.150815:0:2406:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000020:00000001:3.0:1787695820.150818:0:2405:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:2.0:1787695820.150819:0:2406:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1787695820.150822:0:2405:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=21 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000100:00000001:2.0:1787695820.150823:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695820.150826:0:2405:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1787695820.150828:0:2405:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b14517d5a00. 02000000:00000001:3.0:1787695820.150832:0:2405:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787695820.150834:0:2405:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1787695820.150841:0:2405:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466fdc00. 00000100:00000001:3.0:1787695820.150844:0:2405:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787695820.150845:0:2405:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695820.150847:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695820.342565:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695820.342571:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695820.342584:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695820.342585:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695820.342592:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695820.342593:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695820.342600:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695820.342601:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695820.342603:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695820.342604:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695820.342606:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695820.342607:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695820.342680:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695820.342697:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695820.342700:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787695820.662428:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695820.662437:0:2404:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000400:0.0:1787695820.662453:0:2404:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695735/real 1787695735] req@ffff8b1446664a80 x1874531139514368/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695820.662473:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695820.662475:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695820.662479:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547020b0. 00000400:00000200:0.0:1787695820.662485:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46088 00000400:00000010:0.0:1787695820.662488:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46088. 00000100:00000001:0.0:1787695820.662492:0:2404:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695820.662497:0:2404:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446664a80 x1874531139514368/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695820.662506:0:2404:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446664a80 x1874531139514368/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.662536:0:2404:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695820.662538:0:2404:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.662542:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695820.662545:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695820.662550:0:2404:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446664a80 x1874531139514368/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.662558:0:2404:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:0.0:1787695820.662561:0:2404:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b1458081800 already not connected (conn 22, was 15): DISCONN 00000100:00000001:0.0:1787695820.662565:0:2404:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:0.0:1787695820.662566:0:2404:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.662569:0:2404:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000400:0.0:1787695820.662574:0:2404:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695735/real 1787695735] req@ffff8b1446665880 x1874531139514752/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695820.662590:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695820.662592:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695820.662594:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702f78. 00000400:00000200:0.0:1787695820.662597:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f464c8 00000400:00000010:0.0:1787695820.662599:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f464c8. 00000100:00000001:0.0:1787695820.662602:0:2404:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695820.662605:0:2404:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665880 x1874531139514752/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695820.662612:0:2404:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665880 x1874531139514752/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.662619:0:2404:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695820.662620:0:2404:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.662622:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695820.662623:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695820.662626:0:2404:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446665880 x1874531139514752/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.662632:0:2404:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:0.0:1787695820.662651:0:2404:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b144515c000 already not connected (conn 18, was 11): DISCONN 00000100:00000001:0.0:1787695820.662654:0:2404:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:0.0:1787695820.662656:0:2404:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.662706:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695820.662708:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787695820.662711:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695820.662713:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695820.662717:0:2404:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.662719:0:2404:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695820.662721:0:2404:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695820.662727:0:2404:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446664a80 x1874531139514368/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.662734:0:2404:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:0.0:1787695820.662737:0:2404:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.662739:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695820.662741:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695820.662744:0:2404:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446664a80 x1874531139514368/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695820.662752:0:2404:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446664a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2404:1874531139514368:192.168.201.140@tcp:400:kworker.0 00000100:00000001:0.0:1787695820.662757:0:2404:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.662767:0:2404:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695820.662769:0:2404:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695820.662774:0:2404:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665880 x1874531139514752/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.662780:0:2404:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:0.0:1787695820.662782:0:2404:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.662783:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695820.662784:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695820.662787:0:2404:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665880 x1874531139514752/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695820.662821:0:2404:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:608c526c-eb6e-4a54-80a5-8969828241eb:2404:1874531139514752:192.168.201.140@tcp:400:kworker.0 00000100:00000001:0.0:1787695820.662826:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.662828:0:2404:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695820.662832:0:2404:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446664a80 x1874531139514368/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.662838:0:2404:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695820.662841:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695820.662845:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b144667d400. 02000000:00000001:0.0:1787695820.662849:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695820.662853:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695820.662856:0:2404:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=20 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695820.662858:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695820.662860:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1449817400. 02000000:00000001:0.0:1787695820.662865:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695820.662867:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695820.662869:0:2404:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446664a80. 00000100:00000001:0.0:1787695820.662872:0:2404:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695820.662873:0:2404:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.662875:0:2404:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695820.662878:0:2404:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665880 x1874531139514752/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695820 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.662885:0:2404:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695820.662886:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695820.662887:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b144667cc00. 02000000:00000001:0.0:1787695820.662890:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695820.662891:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695820.662893:0:2404:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=21 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695820.662895:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695820.662897:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1449817300. 02000000:00000001:0.0:1787695820.662899:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695820.662900:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695820.662901:0:2404:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665880. 00000100:00000001:0.0:1787695820.662911:0:2404:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695820.662912:0:2404:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.662916:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695820.663412:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695820.663417:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787695820.663419:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695820.663421:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695820.663424:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695820.663428:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3708 00000400:00000200:3.0:1787695820.663441:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695820.663445:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695820.663448:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695820.663450:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695820.663453:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695820.663455:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695820.663456:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695820.663459:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695820.663460:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:3.0:1787695820.663462:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695820.663480:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787695820.663483:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787695820.663485:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787695820.663487:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695820.663488:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695820.663490:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695820.663491:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695820.663493:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787695820.663495:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695820.663504:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695820.663506:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695820.663508:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695820.663510:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695820.663512:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787695820.663527:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695820.663533:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4a80. 00000020:00000040:3.0:1787695820.663535:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787695820.663538:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695820.663539:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695820.663542:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695820.663544:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695820.663549:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695820.663554:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675800. 00000100:00000001:3.0:1787695820.663559:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695820.663564:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4a80 x1874531139527680/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695820.663572:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695820.663575:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695820.663579:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ad0d67] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4a80 x1874531139527680/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695820.663594:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695820.663597:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695820.663599:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695820.663601:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695820.663602:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695820.663604:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695820.663606:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3708 00000400:00000200:3.0:1787695820.663609:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695820.663612:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695820.663614:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695820.663617:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695820.663621:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695820.663625:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695820.663625:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695820.663629:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1787695820.663630:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695820.663633:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000040:0.0:1787695820.663636:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4a80 x1874531139527680/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695820.663636:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695820.663639:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:3.0:1787695820.663642:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695820.663645:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:3.0:1787695820.663649:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000001:0.0:1787695820.663652:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1787695820.663652:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000001:0.0:1787695820.663654:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695820.663658:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695820.663659:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787695820.663661:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787695820.663663:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1787695820.663665:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695820.663668:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695820.663669:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695820.663673:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00100000:0.0:1787695820.663674:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139527680:192.168.201.140@tcp:250:kworker.0 00000100:00080000:3.0:1787695820.663679:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695820.663680:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787695820.663684:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695820.663685:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787695820.663686:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787695820.663689:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695820.663692:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695820.663693:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787695820.663697:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695820.663699:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f7400. 00000100:00000001:3.0:1787695820.663699:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787695820.663703:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787695820.663705:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695820.663708:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695820.663709:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695820.663711:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695820.663714:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b580. 00000100:00000010:3.0:1787695820.663716:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4000. 00000020:00000040:3.0:1787695820.663719:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000400:00000010:0.0:1787695820.663721:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 02000000:00000001:3.0:1787695820.663722:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695820.663725:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695820.663727:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:0.0:1787695820.663728:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139527680, portal 25 02000000:00000001:3.0:1787695820.663728:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695820.663731:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.663732:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695820.663736:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930146432 : -128555779405184 : ffff8b14466f4a80) 02000000:00000010:3.0:1787695820.663738:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676000. 00000100:00000040:0.0:1787695820.663742:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4a80 x1874531139527680/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695845 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695820.663743:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.663749:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695820.663753:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695820.663756:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00100000:3.0:1787695820.663758:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4000 x1874531139527808/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1787695820.663760:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139527680, offset 0 00000400:00000200:0.0:1787695820.663770:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695820.663770:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695820.663774:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695820.663778:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5a8f356] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4000 x1874531139527808/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695820.663779:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695820.663793:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:3.0:1787695820.663795:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695820.663798:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1787695820.663799:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695820.663805:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00080000:3.0:1787695820.663806:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:0.0:1787695820.663810:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1787695820.663812:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1787695820.663813:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00080000:3.0:1787695820.663818:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1787695820.663820:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16932:16932:256:4294967295] 192.168.201.140@tcp LPNI seq info [16931:16931:8:4294967295] 00000400:00000200:0.0:1787695820.663826:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695820.663826:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695820.663834:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3708 00000400:00000200:0.0:1787695820.663842:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695820.663851:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000400:00000200:3.0:1787695820.663851:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000800:00000010:0.0:1787695820.663857:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000400:00000200:3.0:1787695820.663859:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695820.663863:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000800:00000200:0.0:1787695820.663867:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:3.0:1787695820.663869:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000800:00000200:0.0:1787695820.663872:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 12 secs later) 00000100:00000001:3.0:1787695820.663873:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695820.663878:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695820.663881:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000200:0.1:1787695820.663882:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000020:00000001:3.0:1787695820.663885:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000200:0.1:1787695820.663887:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 12 secs later) 00000020:00000040:3.0:1787695820.663888:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:3.0:1787695820.663891:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000100:0.0:1787695820.663894:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000020:00000040:3.0:1787695820.663895:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000800:00000010:0.0:1787695820.663898:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000020:00000001:3.0:1787695820.663900:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000400:00000200:0.0:1787695820.663903:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:3.0:1787695820.663905:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000400:00000200:0.0:1787695820.663909:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695820.663914:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000100:00000040:3.0:1787695820.663915:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000010:0.0:1787695820.663917:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:3.0:1787695820.663922:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695820.663923:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695820.663930:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4a80 x1874531139527680/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695845 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.663936:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695820.663938:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000040:0.0:1787695820.663939:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4a80 x1874531139527680/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695845 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.663945:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.663946:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695820.663950:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695820.663954:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:0.0:1787695820.663956:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695820.663961:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298392377 00000020:00000040:3.0:1787695820.663961:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:3.0:1787695820.663968:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695820.663970:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298387256 00000100:00000001:3.0:1787695820.663972:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695820.663974:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695820.663976:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695820.663981:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.663982:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298392377 00000100:00000001:3.0:1787695820.663983:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695820.663987:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787695820.663988:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298387256 00000008:00000001:3.0:1787695820.663990:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.663994:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695820.663994:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.663998:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695820.663998:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4700. 00000020:00000040:3.0:1787695820.664004:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=21 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000100:00000001:0.0:1787695820.664005:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1787695820.664010:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:0.0:1787695820.664011:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695820/real 1787695820] req@ffff8b14466f4a80 x1874531139527680/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695820.664012:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695820.664017:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695820.664019:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695820.664023:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787695820.664028:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000010:3.0:1787695820.664031:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 00000400:00000001:0.0:1787695820.664032:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695820.664034:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b580. 00000100:00000001:3.0:1787695820.664037:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695820.664043:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4700 x1874531139527936/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1787695820.664047:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000100:00000001:3.0:1787695820.664051:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000010:0.0:1787695820.664054:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:3.0:1787695820.664055:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.664058:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:3.0:1787695820.664063:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008818c2cc] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4700 x1874531139527936/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695820.664066:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4a80 x1874531139527680/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695820.664083:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695820.664087:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4a80 x1874531139527680/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695820.664087:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695820.664091:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787695820.664094:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695820.664097:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695820.664099:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695820.664100:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.664102:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695820.664102:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695820.664105:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4a80 x1874531139527680/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.664112:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695820.664118:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:0.0:1787695820.664121:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4a80 x1874531139527680/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695820.664122:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3708 00000400:00000200:3.0:1787695820.664129:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695820.664136:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:0.0:1787695820.664139:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787695820.664140:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787695820.664145:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695820.664146:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695820.664149:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695820.664150:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695820.664152:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695820.664153:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695820.664155:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787695820.664157:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1787695820.664158:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:3.0:1787695820.664161:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787695820.664163:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000020:00000040:3.0:1787695820.664164:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000100:00000001:0.0:1787695820.664166:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1787695820.664167:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00080000:0.0:1787695820.664173:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:3.0:1787695820.664173:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:3.0:1787695820.664177:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00080000:0.0:1787695820.664181:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:3.0:1787695820.664181:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695820.664185:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695820.664186:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787695820.664189:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695820.664194:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00000040:0.0:1787695820.664195:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4a80 x1874531139527680/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695820.664200:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695820.664205:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695820.664209:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695820.664213:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695820.664216:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695820.664217:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695820.664219:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787695820.664220:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695820.664221:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139527680:192.168.201.140@tcp:250:kworker.0 00000100:00000001:3.0:1787695820.664222:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695820.664228:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5500. 00000100:00000001:0.0:1787695820.664231:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.664234:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000040:3.0:1787695820.664236:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000100:00000040:0.0:1787695820.664242:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4a80 x1874531139527680/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695820.664242:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695820.664244:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695820.664249:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695820.664251:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695820.664253:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695820.664258:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675400. 00000100:00000001:0.0:1787695820.664260:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1787695820.664262:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695820.664264:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695820.664268:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f7400. 00000100:00100000:3.0:1787695820.664269:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5500 x1874531139528064/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787695820.664272:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695820.664275:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:3.0:1787695820.664279:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695820.664280:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695820.664284:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a777d68] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5500 x1874531139528064/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1787695820.664285:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:0.0:1787695820.664291:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695820.664293:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675800. 00000100:00000001:3.0:1787695820.664299:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695820.664300:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695820.664303:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:3.0:1787695820.664305:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000010:0.0:1787695820.664306:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4a80. 00000100:00000001:0.0:1787695820.664311:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00080000:3.0:1787695820.664313:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1787695820.664314:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.664317:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695820.664318:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787695820.664322:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1787695820.664323:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695820.664325:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695820.664331:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787695820.664332:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695820.664336:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1787695820.664339:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3708 00000100:00000001:0.0:1787695820.664341:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695820.664349:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4000 x1874531139527808/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695820.664349:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695820.664354:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695820.664359:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695820.664390:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695820.664392:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695820.664394:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695820.664395:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695820.664400:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695820.664403:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000100:00000001:0.0:1787695820.664404:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695820.664406:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695820.664408:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695820.664412:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1787695820.664412:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1787695820.664416:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695820.664416:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:3.0:1787695820.664422:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00100000:0.0:1787695820.664425:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139527808:192.168.201.140@tcp:38:kworker.0 00000020:00000001:3.0:1787695820.664426:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000001:0.0:1787695820.664432:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695820.664435:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1787695820.664436:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695820.664438:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695820.664439:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787695820.664441:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1787695820.664443:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695820.664446:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000010:0.0:1787695820.664447:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f7400. 02000000:00000001:0.0:1787695820.664452:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695820.664452:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:3.0:1787695820.664458:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:0.0:1787695820.664462:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695820.664463:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695820.664465:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b580. 02000000:00000001:3.0:1787695820.664466:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:0.0:1787695820.664470:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 02000000:00000001:3.0:1787695820.664471:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:0.0:1787695820.664475:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139527808, portal 10 02000000:00000001:3.0:1787695820.664475:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.664480:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787695820.664481:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787695820.664484:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930143744 : -128555779407872 : ffff8b14466f4000) 00000008:00100000:3.0:1787695820.664485:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787695820.664490:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695820.664491:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695820.664494:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4000 x1874531139527808/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695845 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1787695820.664496:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5f80. 00000020:00000040:3.0:1787695820.664501:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:3.0:1787695820.664506:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787695820.664507:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787695820.664509:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695820.664513:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695820.664526:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695820.664529:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695820.664530:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695820.664534:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 02000000:00000010:3.0:1787695820.664536:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 00000100:00000200:0.0:1787695820.664542:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139527808, offset 0 00000100:00000001:3.0:1787695820.664542:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695820.664550:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5f80 x1874531139528192/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1787695820.664556:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695820.664562:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1787695820.664566:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695820.664567:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695820.664572:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006e7bde76] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5f80 x1874531139528192/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695820.664575:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695820.664582:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695820.664586:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695820.664591:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695820.664595:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:3.0:1787695820.664596:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695820.664600:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695820.664605:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695820.664608:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1787695820.664609:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16933:16933:256:4294967295] 192.168.201.140@tcp LPNI seq info [16932:16932:8:4294967295] 00000100:00080000:3.0:1787695820.664610:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695820.664616:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:0.0:1787695820.664619:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:3.0:1787695820.664622:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3708 00000400:00000200:0.0:1787695820.664629:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695820.664634:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000400:00000200:3.0:1787695820.664637:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000800:00000010:0.0:1787695820.664639:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000400:00000200:3.0:1787695820.664643:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000800:00000200:0.0:1787695820.664649:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00080000:3.0:1787695820.664649:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000800:00000200:0.0:1787695820.664655:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 12 secs later) 00000100:00000040:3.0:1787695820.664656:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695820.664661:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.1:1787695820.664664:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:3.0:1787695820.664666:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000200:0.1:1787695820.664668:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 12 secs later) 00000100:00000001:3.0:1787695820.664669:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695820.664675:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000100:0.0:1787695820.664678:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000020:00000040:3.0:1787695820.664678:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:3.0:1787695820.664682:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000010:0.0:1787695820.664683:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000020:00000040:3.0:1787695820.664686:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000400:00000200:0.0:1787695820.664687:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:3.0:1787695820.664691:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000200:0.0:1787695820.664696:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000020:00000001:3.0:1787695820.664697:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000200:0.0:1787695820.664700:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000100:00000040:3.0:1787695820.664702:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000010:0.0:1787695820.664703:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:3.0:1787695820.664706:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695820.664708:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1787695820.664711:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695820.664713:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000200:0.0:1787695820.664716:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4000 x1874531139527808/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695845 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1787695820.664717:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:3.0:1787695820.664722:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695820.664728:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.664732:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695820.664737:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4000 x1874531139527808/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695845 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695820.664737:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695820.664742:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695820.664747:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.664748:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.664753:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787695820.664753:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695820.664758:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787695820.664759:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787695820.664762:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695820.664765:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000100:00000001:3.0:1787695820.664765:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695820.664770:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6d80. 00000400:00000200:0.0:1787695820.664771:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000040:3.0:1787695820.664774:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000100:00000001:0.0:1787695820.664775:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298382279 02000000:00000001:3.0:1787695820.664778:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787695820.664780:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298392377 02000000:00000001:3.0:1787695820.664781:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.664784:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298382279 00000100:00000001:3.0:1787695820.664785:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695820.664788:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787695820.664789:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298392377 02000000:00000001:3.0:1787695820.664791:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.664792:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:3.0:1787695820.664796:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 00000100:00000001:3.0:1787695820.664799:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.664804:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.664806:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787695820.664810:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6d80 x1874531139528320/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787695820.664812:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695820/real 1787695820] req@ffff8b14466f4000 x1874531139527808/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695820.664839:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787695820.664841:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000001:0.0:1787695820.664842:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695820.664845:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b580. 00000100:00000001:3.0:1787695820.664846:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695820.664848:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:0.0:1787695820.664852:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:0.0:1787695820.664854:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:3.0:1787695820.664857:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c3be57b] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6d80 x1874531139528320/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695820.664860:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4000 x1874531139527808/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695820.664881:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695820.664885:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4000 x1874531139527808/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695820.664886:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695820.664895:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695820.664899:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787695820.664901:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:3.0:1787695820.664903:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695820.664904:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.664907:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695820.664910:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695820.664911:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:0.0:1787695820.664914:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4000 x1874531139527808/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695820.664918:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3708 00000100:00000001:0.0:1787695820.664924:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787695820.664930:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:0.0:1787695820.664931:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4000 x1874531139527808/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.664939:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:3.0:1787695820.664940:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:0.0:1787695820.664942:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.664946:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:3.0:1787695820.664946:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787695820.664947:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.664949:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695820.664950:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695820.664952:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000040:3.0:1787695820.664961:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00080000:0.0:1787695820.664966:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:3.0:1787695820.664966:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695820.664971:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1787695820.664972:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695820.664976:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000040:0.0:1787695820.664980:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4000 x1874531139527808/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695820.664981:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695820.664983:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000020:00000001:3.0:1787695820.664986:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695820.664990:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:3.0:1787695820.664994:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00100000:0.0:1787695820.664995:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139527808:192.168.201.140@tcp:38:kworker.0 00000020:00000001:3.0:1787695820.664998:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:0.0:1787695820.665001:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1787695820.665004:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:0.0:1787695820.665007:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4700 x1874531139527936/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695820.665008:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695820.665014:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695820.665018:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695820.665022:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00000001:0.0:1787695820.665024:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1787695820.665027:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695820.665028:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695820.665032:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787695820.665033:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695820.665035:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695820.665036:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787695820.665040:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695820.665041:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1787695820.665045:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139527936:192.168.201.140@tcp:38:kworker.0 02000000:00000001:3.0:1787695820.665046:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695820.665050:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787695820.665052:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:3.0:1787695820.665053:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787695820.665055:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:3.0:1787695820.665058:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695820.665059:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695820.665062:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695820.665063:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:3.0:1787695820.665067:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4380. 02000000:00000010:0.0:1787695820.665068:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5c00. 00000020:00000040:3.0:1787695820.665071:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=22 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:0.0:1787695820.665072:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695820.665076:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1787695820.665080:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787695820.665080:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695820.665084:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b580. 00000100:00000001:3.0:1787695820.665085:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695820.665087:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:0.0:1787695820.665090:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 02000000:00000001:3.0:1787695820.665091:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695820.665095:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139527936, portal 10 02000000:00000010:3.0:1787695820.665098:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 00000100:00000001:0.0:1787695820.665099:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695820.665100:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930145536 : -128555779406080 : ffff8b14466f4700) 00000100:00000040:0.0:1787695820.665106:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4700 x1874531139527936/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695845 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695820.665109:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.665115:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1787695820.665118:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4380 x1874531139528448/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1787695820.665129:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695820.665130:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000010:0.0:1787695820.665133:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000001:3.0:1787695820.665133:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695820.665139:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139527936, offset 0 00000100:00000040:3.0:1787695820.665143:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d2982c15] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4380 x1874531139528448/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695820.665147:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695820.665155:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695820.665160:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695820.665162:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000040:3.0:1787695820.665164:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1787695820.665167:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1787695820.665169:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:0.0:1787695820.665172:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000040:3.0:1787695820.665173:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1787695820.665176:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000040:3.0:1787695820.665178:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:0.0:1787695820.665181:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000040:3.0:1787695820.665184:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3718) 00000400:00000200:0.0:1787695820.665187:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16934:16934:256:4294967295] 192.168.201.140@tcp LPNI seq info [16933:16933:8:4294967295] 00000400:00000200:0.0:1787695820.665194:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695820.665201:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695820.665206:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695820.665210:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:0.0:1787695820.665214:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695820.665217:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 12 secs later) 00000800:00000200:0.1:1787695820.665222:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695820.665225:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 12 secs later) 00000800:00000100:0.0:1787695820.665228:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695820.665231:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:0.0:1787695820.665234:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695820.665237:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695820.665241:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695820.665243:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695820.665245:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695820.665250:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4700 x1874531139527936/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695845 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.665259:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695820.665262:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4700 x1874531139527936/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695845 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.665269:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.665271:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695820.665281:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695820.665284:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:0.0:1787695820.665287:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695820.665290:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298381127 00000100:00000001:0.0:1787695820.665292:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298392377 00000100:00000001:0.0:1787695820.665294:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298381127 00000100:00000001:0.0:1787695820.665296:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298392377 00000100:00000001:0.0:1787695820.665298:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695820.665301:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.665303:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695820.665307:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695820/real 1787695820] req@ffff8b14466f4700 x1874531139527936/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695820.665316:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695820.665317:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695820.665319:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b580. 00000400:00000200:0.0:1787695820.665321:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:0.0:1787695820.665323:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:0.0:1787695820.665325:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695820.665328:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4700 x1874531139527936/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695820.665338:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4700 x1874531139527936/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.665345:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695820.665347:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.665348:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695820.665350:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695820.665354:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4700 x1874531139527936/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.665361:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695820.665385:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4700 x1874531139527936/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.665399:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695820.665402:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.665404:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695820.665405:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.665408:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695820.665410:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695820.665413:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695820.665418:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695820.665421:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695820.665427:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4700 x1874531139527936/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695820.665436:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139527936:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695820.665440:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695820.665442:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5500 x1874531139528064/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.665447:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695820.665448:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695820.665450:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695820.665451:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695820.665452:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695820.665454:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139528064:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695820.665456:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695820.665457:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695820.665459:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695820.665460:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695820.665463:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5800. 02000000:00000001:0.0:1787695820.665464:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695820.665467:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695820.665488:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b580. 00000400:00000010:0.0:1787695820.665491:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 00000100:00000200:0.0:1787695820.665493:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139528064, portal 4 00000100:00000001:0.0:1787695820.665495:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695820.665496:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930149120 : -128555779402496 : ffff8b14466f5500) 00000100:00000040:0.0:1787695820.665501:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5500 x1874531139528064/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695845 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.665506:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695820.665507:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695820.665509:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695820.665510:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139528064, offset 0 00000400:00000200:0.0:1787695820.665513:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695820.665528:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695820.665532:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695820.665535:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695820.665537:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695820.665540:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695820.665543:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695820.665547:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16935:16935:256:4294967295] 192.168.201.140@tcp LPNI seq info [16934:16934:8:4294967295] 00000400:00000200:0.0:1787695820.665551:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695820.665559:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695820.665564:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695820.665568:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:0.0:1787695820.665573:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695820.665576:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 12 secs later) 00000800:00000200:0.1:1787695820.665580:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695820.665581:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 12 secs later) 00000800:00000100:0.0:1787695820.665584:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695820.665586:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:0.0:1787695820.665588:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695820.665590:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695820.665592:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695820.665594:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695820.665600:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695820.665603:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5500 x1874531139528064/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695845 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.665609:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695820.665611:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5500 x1874531139528064/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695845 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.665616:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.665617:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695820.665620:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695820.665622:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:0.0:1787695820.665624:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695820.665625:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298381158 00000100:00000001:0.0:1787695820.665627:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298392377 00000100:00000001:0.0:1787695820.665629:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298381158 00000100:00000001:0.0:1787695820.665630:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298392377 00000100:00000001:0.0:1787695820.665632:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695820.665633:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.665634:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695820.665636:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695820/real 1787695820] req@ffff8b14466f5500 x1874531139528064/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695820.665641:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695820.665642:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695820.665643:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b580. 00000400:00000200:0.0:1787695820.665645:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:0.0:1787695820.665646:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:0.0:1787695820.665648:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695820.665650:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5500 x1874531139528064/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695820.665655:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5500 x1874531139528064/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.665668:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695820.665669:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.665670:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695820.665671:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695820.665674:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5500 x1874531139528064/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.665678:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695820.665681:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5500 x1874531139528064/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.665685:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695820.665687:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.665688:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695820.665689:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.665690:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695820.665691:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695820.665692:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695820.665695:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695820.665696:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695820.665698:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5500 x1874531139528064/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695820.665703:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139528064:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695820.665705:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.665707:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695820.665709:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4000 x1874531139527808/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.665713:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695820.665714:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695820.665719:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f7400. 02000000:00000001:0.0:1787695820.665721:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695820.665722:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695820.665723:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695820.665724:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695820.665726:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676000. 02000000:00000001:0.0:1787695820.665728:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695820.665729:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695820.665730:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4000. 00000100:00000001:0.0:1787695820.665732:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695820.665733:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.665734:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695820.665737:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4700 x1874531139527936/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.665741:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695820.665741:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695820.665742:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5c00. 02000000:00000001:0.0:1787695820.665744:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695820.665745:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695820.665746:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=20 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695820.665747:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695820.665748:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:0.0:1787695820.665751:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695820.665752:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695820.665753:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4700. 00000100:00000001:0.0:1787695820.665754:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695820.665754:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.665755:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695820.665757:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5500 x1874531139528064/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.665761:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695820.665762:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695820.665762:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5800. 02000000:00000001:0.0:1787695820.665764:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695820.665764:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695820.665765:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:0.0:1787695820.665769:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695820.665770:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675400. 02000000:00000001:0.0:1787695820.665772:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695820.665773:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695820.665774:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5500. 00000100:00000001:0.0:1787695820.665775:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695820.665776:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.665777:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.665780:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695820.665781:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.665782:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695820.665783:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695820.665784:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695820.665786:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5f80 x1874531139528192/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.665791:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695820.665792:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695820.665793:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695820.665795:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695820.665796:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695820.665798:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139528192:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695820.665800:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695820.665801:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695820.665802:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695820.665803:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695820.665805:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5800. 02000000:00000001:0.0:1787695820.665806:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695820.665809:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695820.665811:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b580. 00000400:00000010:0.0:1787695820.665814:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 00000100:00000200:0.0:1787695820.665817:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139528192, portal 4 00000100:00000001:0.0:1787695820.665819:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695820.665821:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930151808 : -128555779399808 : ffff8b14466f5f80) 00000100:00000040:0.0:1787695820.665827:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5f80 x1874531139528192/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695845 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.665845:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695820.665848:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695820.665851:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695820.665854:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139528192, offset 0 00000400:00000200:0.0:1787695820.665858:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695820.665862:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695820.665866:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695820.665868:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695820.665870:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695820.665872:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695820.665873:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695820.665875:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16936:16936:256:4294967295] 192.168.201.140@tcp LPNI seq info [16935:16935:8:4294967295] 00000400:00000200:0.0:1787695820.665878:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695820.665883:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695820.665886:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695820.665889:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:0.0:1787695820.665892:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695820.665894:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 12 secs later) 00000800:00000200:0.1:1787695820.665896:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695820.665898:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 12 secs later) 00000800:00000100:0.0:1787695820.665901:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695820.665903:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:0.0:1787695820.665907:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695820.665910:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695820.665913:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695820.665914:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695820.665916:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695820.665920:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5f80 x1874531139528192/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695845 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.665928:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695820.665931:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5f80 x1874531139528192/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695845 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.665950:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.665952:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695820.665956:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695820.665960:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:0.0:1787695820.665964:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695820.665966:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298380830 00000100:00000001:0.0:1787695820.665968:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298392377 00000100:00000001:0.0:1787695820.665970:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298380830 00000100:00000001:0.0:1787695820.665971:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298392377 00000100:00000001:0.0:1787695820.665973:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695820.665975:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.665976:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695820.665979:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695820/real 1787695820] req@ffff8b14466f5f80 x1874531139528192/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695820.665984:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695820.665985:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695820.665987:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b580. 00000400:00000200:0.0:1787695820.665990:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:0.0:1787695820.665991:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:0.0:1787695820.665993:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695820.665995:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5f80 x1874531139528192/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695820.666001:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5f80 x1874531139528192/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.666005:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695820.666006:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.666007:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695820.666008:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695820.666010:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5f80 x1874531139528192/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.666019:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695820.666022:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5f80 x1874531139528192/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.666026:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695820.666027:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.666028:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695820.666029:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.666031:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695820.666032:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695820.666033:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695820.666036:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695820.666037:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695820.666040:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5f80 x1874531139528192/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695820.666044:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139528192:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695820.666047:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695820.666049:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6d80 x1874531139528320/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.666053:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695820.666054:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695820.666055:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695820.666056:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695820.666058:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695820.666060:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139528320:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695820.666062:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695820.666063:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695820.666067:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695820.666068:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695820.666070:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5c00. 02000000:00000001:0.0:1787695820.666071:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695820.666074:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695820.666075:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b580. 00000400:00000010:0.0:1787695820.666078:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 00000100:00000200:0.0:1787695820.666080:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139528320, portal 10 00000100:00000001:0.0:1787695820.666081:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695820.666082:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930155392 : -128555779396224 : ffff8b14466f6d80) 00000100:00000040:0.0:1787695820.666086:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6d80 x1874531139528320/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695845 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.666090:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695820.666092:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695820.666093:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695820.666095:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139528320, offset 0 00000400:00000200:0.0:1787695820.666097:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695820.666100:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695820.666102:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695820.666104:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695820.666106:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695820.666108:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695820.666109:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695820.666111:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16937:16937:256:4294967295] 192.168.201.140@tcp LPNI seq info [16936:16936:8:4294967295] 00000400:00000200:0.0:1787695820.666115:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695820.666119:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695820.666122:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695820.666124:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:0.0:1787695820.666127:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695820.666129:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 12 secs later) 00000800:00000200:0.1:1787695820.666131:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695820.666137:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 12 secs later) 00000800:00000100:0.0:1787695820.666140:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695820.666144:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:0.0:1787695820.666146:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695820.666149:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695820.666151:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695820.666152:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695820.666153:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695820.666156:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6d80 x1874531139528320/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695845 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.666161:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695820.666163:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6d80 x1874531139528320/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695845 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.666167:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.666168:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695820.666170:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695820.666172:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:0.0:1787695820.666174:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695820.666176:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298381016 00000100:00000001:0.0:1787695820.666178:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298392377 00000100:00000001:0.0:1787695820.666180:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298381016 00000100:00000001:0.0:1787695820.666182:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298392377 00000100:00000001:0.0:1787695820.666185:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695820.666187:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.666189:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695820.666193:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695820/real 1787695820] req@ffff8b14466f6d80 x1874531139528320/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695820.666201:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695820.666202:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695820.666204:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b580. 00000400:00000200:0.0:1787695820.666206:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:0.0:1787695820.666208:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:0.0:1787695820.666216:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695820.666220:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6d80 x1874531139528320/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695820.666229:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6d80 x1874531139528320/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.666236:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695820.666238:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.666239:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695820.666240:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695820.666243:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6d80 x1874531139528320/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.666251:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695820.666256:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6d80 x1874531139528320/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.666262:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695820.666265:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.666266:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695820.666267:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.666269:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695820.666271:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695820.666274:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695820.666277:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695820.666280:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695820.666284:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6d80 x1874531139528320/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695820.666292:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139528320:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695820.666296:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695820.666305:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4380 x1874531139528448/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.666313:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695820.666315:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695820.666317:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695820.666319:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695820.666321:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695820.666324:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139528448:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695820.666328:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695820.666330:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695820.666332:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695820.666334:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695820.666337:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f7400. 02000000:00000001:0.0:1787695820.666339:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695820.666343:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695820.666345:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b580. 00000400:00000010:0.0:1787695820.666349:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 00000100:00000200:0.0:1787695820.666352:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139528448, portal 10 00000100:00000001:0.0:1787695820.666354:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695820.666355:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930144640 : -128555779406976 : ffff8b14466f4380) 00000100:00000040:0.0:1787695820.666361:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4380 x1874531139528448/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695845 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.666391:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695820.666395:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695820.666399:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695820.666402:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139528448, offset 0 00000400:00000200:0.0:1787695820.666407:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695820.666413:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695820.666417:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695820.666421:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695820.666424:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695820.666427:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695820.666434:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695820.666437:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16938:16938:256:4294967295] 192.168.201.140@tcp LPNI seq info [16937:16937:8:4294967295] 00000400:00000200:0.0:1787695820.666440:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695820.666445:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695820.666448:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695820.666450:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:0.0:1787695820.666453:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695820.666455:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 12 secs later) 00000800:00000200:0.1:1787695820.666458:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695820.666460:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 12 secs later) 00000800:00000100:0.0:1787695820.666463:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695820.666464:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:0.0:1787695820.666466:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695820.666468:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695820.666470:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695820.666472:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695820.666474:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695820.666476:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4380 x1874531139528448/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695845 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.666482:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695820.666484:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4380 x1874531139528448/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695845 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.666488:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.666489:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695820.666491:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695820.666493:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:0.0:1787695820.666495:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695820.666497:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298380995 00000100:00000001:0.0:1787695820.666499:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298392377 00000100:00000001:0.0:1787695820.666500:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298380995 00000100:00000001:0.0:1787695820.666502:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298392377 00000100:00000001:0.0:1787695820.666506:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695820.666508:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.666509:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695820.666512:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695820/real 1787695820] req@ffff8b14466f4380 x1874531139528448/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695820.666534:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695820.666535:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695820.666537:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b580. 00000400:00000200:0.0:1787695820.666539:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:0.0:1787695820.666541:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:0.0:1787695820.666543:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695820.666547:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4380 x1874531139528448/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695820.666557:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4380 x1874531139528448/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.666564:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695820.666566:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.666568:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695820.666569:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695820.666573:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4380 x1874531139528448/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.666579:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695820.666582:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4380 x1874531139528448/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.666587:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695820.666589:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.666590:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695820.666591:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.666592:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695820.666593:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695820.666599:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695820.666601:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695820.666603:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695820.666606:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4380 x1874531139528448/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695820.666611:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139528448:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695820.666614:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.666615:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695820.666617:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5f80 x1874531139528192/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.666621:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695820.666622:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695820.666624:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5800. 02000000:00000001:0.0:1787695820.666626:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695820.666627:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695820.666628:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:0.0:1787695820.666629:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695820.666631:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:0.0:1787695820.666633:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695820.666634:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695820.666635:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5f80. 00000100:00000001:0.0:1787695820.666637:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695820.666637:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.666639:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695820.666641:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6d80 x1874531139528320/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.666644:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695820.666645:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695820.666646:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5c00. 02000000:00000001:0.0:1787695820.666648:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695820.666649:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695820.666653:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695820.666655:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695820.666656:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:0.0:1787695820.666658:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695820.666659:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695820.666660:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6d80. 00000100:00000001:0.0:1787695820.666661:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695820.666662:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.666663:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695820.666665:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4380 x1874531139528448/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695845 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695820.666670:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695820.666670:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695820.666671:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f7400. 02000000:00000001:0.0:1787695820.666673:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695820.666673:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695820.666674:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=21 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695820.666675:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695820.666676:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:0.0:1787695820.666678:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695820.666678:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695820.666679:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4380. 00000100:00000001:0.0:1787695820.666681:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695820.666681:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.666682:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695820.666685:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695820.666686:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695820.666687:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695820.666688:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695820.726461:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695820.726467:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695820.726468:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695820.726473:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695820.726475:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695820.726478:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695820.726480:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695820.726482:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695820.726484:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695821.174469:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695821.174482:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695821.174485:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695821.174492:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695821.174495:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695821.174504:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695821.174522:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695821.174545:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695821.174559:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695821.174561:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695821.174565:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695821.174567:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695821.174572:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695821.174575:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695821.366444:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695821.366450:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695821.366455:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695821.366456:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695821.366459:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695821.366460:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695821.366462:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695821.366463:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695821.366466:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695821.366468:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695821.366470:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695821.366475:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695821.366477:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695821.366478:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695821.366480:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695821.366497:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695821.366498:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695821.366500:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695821.366523:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695821.366524:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695821.366578:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695821.366582:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695821.366584:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695821.750464:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695821.750475:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695821.750477:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695821.750494:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695821.750509:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695821.750512:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695821.750514:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695821.750516:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695821.750518:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695822.198484:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695822.198484:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695822.198509:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695822.198509:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695822.198512:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695822.198512:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695822.198518:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695822.198519:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695822.198522:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695822.198522:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695822.198532:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695822.198533:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695822.198535:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695822.198536:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695822.390449:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695822.390454:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695822.390459:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695822.390460:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695822.390462:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695822.390463:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695822.390466:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695822.390467:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695822.390470:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695822.390470:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695822.390473:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695822.390474:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695822.390533:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695822.390538:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695822.390542:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695822.774438:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695822.774447:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695822.774450:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695822.774455:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695822.774459:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695822.774463:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695822.774465:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695822.774468:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695822.774471:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695823.222440:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695823.222460:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695823.222461:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695823.222464:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695823.222482:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695823.222486:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695823.222489:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695823.222492:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695823.222492:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695823.222505:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695823.222505:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695823.222512:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695823.222520:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695823.222524:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695823.414428:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695823.414436:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695823.414438:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695823.414441:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695823.414445:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695823.414448:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695823.414449:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695823.414451:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695823.414454:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695823.414455:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695823.414456:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695823.414458:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695823.414459:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695823.414461:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695823.414462:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695823.414464:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695823.414487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695823.414488:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695823.414491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695823.414492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695823.414494:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695823.414510:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695823.414513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695823.414514:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695823.414559:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695823.414564:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695823.414567:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695823.798517:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695823.798528:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695823.798530:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695823.798543:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695823.798546:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695823.798551:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695823.798553:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695823.798581:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695823.798583:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695824.246479:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695824.246479:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695824.246488:0:2405:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787695824.246492:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695824.246494:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695824.246499:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695824.246502:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000400:3.0:1787695824.246502:0:2405:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695739/real 1787695739] req@ffff8b14466fc380 x1874531139516032/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695824 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695824.246508:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695824.246514:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787695824.246520:0:2405:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787695824.246523:0:2405:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1787695824.246526:0:2405:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14456d10b0. 00000400:00000200:3.0:1787695824.246533:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b145078ac38 00000400:00000010:3.0:1787695824.246536:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b145078ac38. 00000100:00000001:3.0:1787695824.246540:0:2405:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787695824.246544:0:2405:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466fc380 x1874531139516032/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695824 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1787695824.246552:0:2405:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466fc380 x1874531139516032/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695824 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695824.246558:0:2405:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787695824.246559:0:2405:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695824.246562:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695824.246563:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787695824.246567:0:2405:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b14466fc380 x1874531139516032/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695824 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695824.246571:0:2405:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:3.0:1787695824.246573:0:2405:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b144515c000 already not connected (conn 19, was 11): DISCONN 00000100:00000001:3.0:1787695824.246576:0:2405:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:3.0:1787695824.246577:0:2405:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695824.246579:0:2405:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000400:3.0:1787695824.246584:0:2405:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695739/real 1787695739] req@ffff8b14466fc000 x1874531139515520/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695824 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787695824.246590:0:2405:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787695824.246591:0:2405:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1787695824.246606:0:2405:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14456d1000. 00000400:00000200:3.0:1787695824.246608:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b145078a198 00000400:00000010:3.0:1787695824.246609:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b145078a198. 00000100:00000001:3.0:1787695824.246611:0:2405:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787695824.246613:0:2405:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466fc000 x1874531139515520/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695824 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1787695824.246619:0:2405:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466fc000 x1874531139515520/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695824 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695824.246623:0:2405:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787695824.246624:0:2405:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695824.246625:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695824.246626:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787695824.246629:0:2405:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b14466fc000 x1874531139515520/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695824 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695824.246632:0:2405:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:3.0:1787695824.246633:0:2405:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b1458081800 already not connected (conn 23, was 15): DISCONN 00000100:00000001:3.0:1787695824.246635:0:2405:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:3.0:1787695824.246636:0:2405:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695824.246639:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695824.246641:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695824.246642:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695824.246643:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695824.246647:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695824.246649:0:2405:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695824.246651:0:2405:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695824.246654:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466fc380 x1874531139516032/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695824 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695824.246659:0:2405:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:3.0:1787695824.246661:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695824.246662:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695824.246664:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695824.246667:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466fc380 x1874531139516032/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695824 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695824.246683:0:2405:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466fc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:608c526c-eb6e-4a54-80a5-8969828241eb:2405:1874531139516032:192.168.201.140@tcp:400:kworker.0 00000100:00000001:3.0:1787695824.246687:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695824.246689:0:2405:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695824.246690:0:2405:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695824.246694:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466fc000 x1874531139515520/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695824 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695824.246698:0:2405:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:3.0:1787695824.246700:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695824.246701:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695824.246702:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695824.246705:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466fc000 x1874531139515520/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695824 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695824.246712:0:2405:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466fc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2405:1874531139515520:192.168.201.140@tcp:400:kworker.0 00000100:00000001:3.0:1787695824.246716:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695824.246718:0:2405:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695824.246721:0:2405:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466fc380 x1874531139516032/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695824 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695824.246726:0:2405:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787695824.246729:0:2405:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1787695824.246732:0:2405:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1448929c00. 02000000:00000001:3.0:1787695824.246735:0:2405:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787695824.246737:0:2405:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1787695824.246738:0:2405:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=20 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:3.0:1787695824.246740:0:2405:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1787695824.246742:0:2405:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1461888800. 02000000:00000001:3.0:1787695824.246745:0:2405:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787695824.246746:0:2405:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1787695824.246748:0:2405:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466fc380. 00000100:00000001:3.0:1787695824.246754:0:2405:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787695824.246756:0:2405:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695824.246813:0:2405:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695824.246817:0:2405:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466fc000 x1874531139515520/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695824 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695824.246822:0:2405:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787695824.246824:0:2405:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1787695824.246825:0:2405:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1448929a00. 02000000:00000001:3.0:1787695824.246828:0:2405:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787695824.246829:0:2405:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1787695824.246830:0:2405:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=19 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:3.0:1787695824.246832:0:2405:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1787695824.246833:0:2405:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1461888a00. 02000000:00000001:3.0:1787695824.246836:0:2405:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787695824.246837:0:2405:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1787695824.246838:0:2405:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466fc000. 00000100:00000001:3.0:1787695824.246840:0:2405:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787695824.246841:0:2405:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695824.246843:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695824.438495:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695824.438501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695824.438506:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695824.438507:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695824.438510:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695824.438511:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695824.438513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695824.438514:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695824.438517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695824.438517:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695824.438520:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695824.438521:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695824.438576:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695824.438580:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695824.438598:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695824.822455:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695824.822463:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695824.822464:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695824.822469:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695824.822472:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695824.822475:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695824.822476:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695824.822478:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695824.822480:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695825.270433:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695825.270457:0:2406:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787695825.270462:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000400:2.0:1787695825.270471:0:2406:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695740/real 1787695740] req@ffff8b14465aca80 x1874531139516288/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695825 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695825.270475:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695825.270483:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695825.270489:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695825.270491:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1787695825.270493:0:2406:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695825.270497:0:2406:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787695825.270498:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1787695825.270501:0:2406:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14456d18f0. 00000100:00000001:3.0:1787695825.270503:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695825.270509:0:2406:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b145078a7f8 00000400:00000010:2.0:1787695825.270512:0:2406:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b145078a7f8. 00000100:00000001:2.0:1787695825.270516:0:2406:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695825.270533:0:2406:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465aca80 x1874531139516288/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695825 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695825.270541:0:2406:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465aca80 x1874531139516288/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695825 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695825.270546:0:2406:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695825.270548:0:2406:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695825.270550:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695825.270552:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695825.270556:0:2406:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b14465aca80 x1874531139516288/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695825 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695825.270561:0:2406:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:2.0:1787695825.270564:0:2406:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b1458081800 already not connected (conn 23, was 15): DISCONN 00000100:00000001:2.0:1787695825.270566:0:2406:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:2.0:1787695825.270567:0:2406:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695825.270568:0:2406:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000400:2.0:1787695825.270572:0:2406:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695740/real 1787695740] req@ffff8b14465ad500 x1874531139516672/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695825 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695825.270577:0:2406:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695825.270578:0:2406:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695825.270580:0:2406:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14456d1dc0. 00000400:00000200:2.0:1787695825.270581:0:2406:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b145078add0 00000400:00000010:2.0:1787695825.270583:0:2406:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b145078add0. 00000100:00000001:2.0:1787695825.270584:0:2406:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695825.270586:0:2406:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ad500 x1874531139516672/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695825 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695825.270592:0:2406:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ad500 x1874531139516672/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695825 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695825.270595:0:2406:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695825.270596:0:2406:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695825.270597:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695825.270598:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695825.270604:0:2406:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b14465ad500 x1874531139516672/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695825 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695825.270609:0:2406:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:2.0:1787695825.270610:0:2406:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b144515c000 already not connected (conn 19, was 11): DISCONN 00000100:00000001:2.0:1787695825.270612:0:2406:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:2.0:1787695825.270612:0:2406:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695825.270615:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695825.270616:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695825.270618:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695825.270619:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695825.270621:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695825.270622:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695825.270624:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695825.270627:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465aca80 x1874531139516288/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695825 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695825.270632:0:2406:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:2.0:1787695825.270635:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695825.270636:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695825.270636:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695825.270639:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465aca80 x1874531139516288/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695825 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695825.270644:0:2406:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2406:1874531139516288:192.168.201.140@tcp:400:kworker.0 00000100:00000001:2.0:1787695825.270647:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695825.270648:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695825.270649:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695825.270651:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ad500 x1874531139516672/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695825 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695825.270655:0:2406:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:2.0:1787695825.270657:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695825.270658:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695825.270661:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695825.270664:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ad500 x1874531139516672/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695825 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695825.270669:0:2406:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:608c526c-eb6e-4a54-80a5-8969828241eb:2406:1874531139516672:192.168.201.140@tcp:400:kworker.0 00000100:00000001:2.0:1787695825.270672:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695825.270674:0:2406:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695825.270676:0:2406:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465aca80 x1874531139516288/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695825 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695825.270680:0:2406:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695825.270682:0:2406:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695825.270686:0:2406:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1448928e00. 02000000:00000001:2.0:1787695825.270689:0:2406:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695825.270691:0:2406:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695825.270693:0:2406:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=18 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:2.0:1787695825.270694:0:2406:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695825.270696:0:2406:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1461888f00. 02000000:00000001:2.0:1787695825.270698:0:2406:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695825.270700:0:2406:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695825.270701:0:2406:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465aca80. 00000100:00000001:2.0:1787695825.270704:0:2406:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695825.270705:0:2406:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695825.270706:0:2406:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695825.270708:0:2406:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ad500 x1874531139516672/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695825 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695825.270712:0:2406:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695825.270713:0:2406:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695825.270714:0:2406:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1448929000. 02000000:00000001:2.0:1787695825.270716:0:2406:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695825.270716:0:2406:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695825.270717:0:2406:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=19 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:2.0:1787695825.270718:0:2406:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695825.270719:0:2406:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b144a733900. 02000000:00000001:2.0:1787695825.270722:0:2406:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695825.270725:0:2406:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695825.270727:0:2406:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ad500. 00000100:00000001:2.0:1787695825.270728:0:2406:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695825.270729:0:2406:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695825.270731:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695825.462474:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695825.462479:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695825.462483:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695825.462484:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695825.462486:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695825.462487:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695825.462490:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695825.462491:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695825.462493:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695825.462494:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695825.462497:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695825.462498:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695825.462593:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695825.462599:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695825.462603:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787695825.782420:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695825.782448:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695825.782451:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.782456:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695825.782458:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695825.782463:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.782467:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695825.782530:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695825.782537:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787695825.782540:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695825.782544:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695825.782550:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695825.782560:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3713 00000400:00000200:1.0:1787695825.782569:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695825.782575:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695825.782578:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695825.782589:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695825.782593:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695825.782595:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695825.782598:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695825.782602:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695825.782604:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:1.0:1787695825.782606:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695825.782609:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:1.0:1787695825.782612:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:1.0:1787695825.782615:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:1.0:1787695825.782618:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695825.782620:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695825.782622:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695825.782624:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695825.782627:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:1.0:1787695825.782630:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695825.782633:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695825.782636:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695825.782639:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695825.782643:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695825.782645:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695825.782648:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695825.782654:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446667800. 00000020:00000040:1.0:1787695825.782658:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:1.0:1787695825.782662:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695825.782663:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695825.782666:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695825.782668:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695825.782672:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695825.782678:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545400. 00000100:00000001:1.0:1787695825.782684:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695825.782692:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446667800 x1874531139528576/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695825.782711:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695825.782715:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695825.782721:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009df211f4] to pc [ptlrpcd_rcv+-1] req@ffff8b1446667800 x1874531139528576/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695825.782746:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695825.782750:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695825.782754:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787695825.782756:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695825.782757:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695825.782761:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695825.782765:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3713 00000400:00000200:1.0:1787695825.782770:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695825.782774:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:0.0:1787695825.782775:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1787695825.782778:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787695825.782779:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695825.782783:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695825.782785:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:0.0:1787695825.782792:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446667800 x1874531139528576/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695825.782793:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695825.782800:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695825.782806:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695825.782808:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1787695825.782810:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787695825.782813:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1787695825.782814:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:1.0:1787695825.782818:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1787695825.782820:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:1.0:1787695825.782821:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 02000000:00000001:0.0:1787695825.782825:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1787695825.782826:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 02000000:00000001:0.0:1787695825.782832:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787695825.782832:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00100000:0.0:1787695825.782841:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446667800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139528576:192.168.201.140@tcp:250:kworker.0 00000100:00000040:1.0:1787695825.782841:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695825.782846:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695825.782850:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1787695825.782851:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695825.782855:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:0.0:1787695825.782863:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:1.0:1787695825.782865:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 02000000:00000001:0.0:1787695825.782868:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695825.782871:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000001:0.0:1787695825.782873:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787695825.782878:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695825.782881:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5800. 02000000:00000001:1.0:1787695825.782881:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787695825.782886:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695825.782887:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695825.782892:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695825.782895:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695825.782897:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:0.0:1787695825.782903:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b580. 00000008:00100000:1.0:1787695825.782903:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787695825.782909:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695825.782912:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695825.782914:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 00000100:00000010:1.0:1787695825.782918:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665880. 00000020:00000040:1.0:1787695825.782923:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000100:00000200:0.0:1787695825.782924:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139528576, portal 25 02000000:00000001:1.0:1787695825.782926:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787695825.782929:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1787695825.782930:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.782933:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929568256 : -128555779983360 : ffff8b1446667800) 00000100:00000001:1.0:1787695825.782935:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695825.782937:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695825.782940:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695825.782945:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545800. 00000100:00000040:0.0:1787695825.782946:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446667800 x1874531139528576/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695850 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695825.782949:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695825.782956:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446665880 x1874531139528704/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695825.782974:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787695825.782979:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:0.0:1787695825.782982:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787695825.782983:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695825.782987:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000040:1.0:1787695825.782989:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000716595a9] to pc [ptlrpcd_rcv+-1] req@ffff8b1446665880 x1874531139528704/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695825.782992:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139528576, offset 0 00000400:00000200:0.0:1787695825.783002:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787695825.783002:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695825.783007:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695825.783010:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787695825.783011:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695825.783013:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1787695825.783015:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695825.783017:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:0.0:1787695825.783022:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00080000:1.0:1787695825.783028:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3713 00000400:00000200:0.0:1787695825.783029:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695825.783036:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695825.783039:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695825.783041:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695825.783044:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16939:16939:256:4294967295] 192.168.201.140@tcp LPNI seq info [16938:16938:8:4294967295] 00000400:00000200:0.0:1787695825.783048:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695825.783053:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695825.783057:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695825.783058:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695825.783061:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695825.783064:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695825.783069:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695825.783073:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:0.0:1787695825.783077:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000020:00000001:1.0:1787695825.783079:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695825.783082:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:1.0:1787695825.783086:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000200:0.0:1787695825.783089:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000020:00000040:1.0:1787695825.783090:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:1.0:1787695825.783096:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000800:00000010:0.0:1787695825.783098:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000020:00000001:1.0:1787695825.783099:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:1.0:1787695825.783107:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000800:00000200:0.0:1787695825.783111:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:1.0:1787695825.783112:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.0:1787695825.783119:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 7 secs later) 00000100:00000040:1.0:1787695825.783119:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695825.783125:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695825.783131:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000800:00000200:0.1:1787695825.783131:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695825.783136:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 7 secs later) 00000100:00080000:1.0:1787695825.783139:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000800:00000100:0.0:1787695825.783146:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787695825.783146:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1787695825.783150:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 02000000:00000001:1.0:1787695825.783152:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1787695825.783155:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695825.783160:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695825.783164:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 02000000:00000001:1.0:1787695825.783165:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1787695825.783166:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 02000000:00000001:1.0:1787695825.783172:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.783173:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787695825.783176:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000200:0.0:1787695825.783179:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446667800 x1874531139528576/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695850 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787695825.783181:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787695825.783189:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695825.783194:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.783197:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000010:1.0:1787695825.783200:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446664a80. 00000100:00000040:0.0:1787695825.783204:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446667800 x1874531139528576/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695850 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1787695825.783206:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=19 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:1.0:1787695825.783213:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695825.783221:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.783232:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695825.783233:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787695825.783237:0:2402:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1787695825.783238:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695825.783243:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.783247:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695825.783249:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446546800. 00000400:00000200:0.0:1787695825.783255:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695825.783256:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.783260:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298397496 00000100:00100000:1.0:1787695825.783266:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446664a80 x1874531139528832/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695825.783270:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298392377 00000100:00000001:0.0:1787695825.783278:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298397496 00000100:00000001:0.0:1787695825.783282:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298392377 00000100:00000001:0.0:1787695825.783284:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695825.783287:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.783288:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1787695825.783291:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1787695825.783294:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695825/real 1787695825] req@ffff8b1446667800 x1874531139528576/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695825.783296:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787695825.783307:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:1.0:1787695825.783308:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ad35ba92] to pc [ptlrpcd_rcv+-1] req@ffff8b1446664a80 x1874531139528832/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695825.783309:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695825.783313:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b580. 00000400:00000200:0.0:1787695825.783317:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:0.0:1787695825.783320:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:0.0:1787695825.783324:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787695825.783327:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695825.783330:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446667800 x1874531139528576/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695825.783333:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695825.783345:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000200:0.0:1787695825.783348:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446667800 x1874531139528576/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.783353:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695825.783356:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695825.783356:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787695825.783361:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695825.783397:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695825.783404:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446667800 x1874531139528576/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.783424:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695825.783443:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446667800 x1874531139528576/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.783451:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695825.783454:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.783456:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695825.783457:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.783459:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695825.783461:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695825.783465:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695825.783469:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695825.783472:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695825.783476:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446667800 x1874531139528576/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695825.783486:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446667800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139528576:192.168.201.140@tcp:250:kworker.0 00000100:00000001:0.0:1787695825.783491:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.783494:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695825.783497:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446667800 x1874531139528576/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.783502:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695825.783504:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695825.783507:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5800. 02000000:00000001:0.0:1787695825.783510:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695825.783512:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695825.783513:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:0.0:1787695825.783515:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695825.783517:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545400. 02000000:00000001:0.0:1787695825.783520:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695825.783527:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695825.783528:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446667800. 00000100:00000001:0.0:1787695825.783532:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695825.783533:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.783535:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787695825.783535:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695825.783541:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695825.783542:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787695825.783543:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.783547:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695825.783551:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1787695825.783551:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3713 00000100:00000001:0.0:1787695825.783553:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695825.783560:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665880 x1874531139528704/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695825.783561:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695825.783570:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000400:00000200:1.0:1787695825.783571:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:0.0:1787695825.783572:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695825.783578:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:1.0:1787695825.783578:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 02000000:00000001:0.0:1787695825.783580:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695825.783584:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695825.783587:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00100000:0.0:1787695825.783589:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139528704:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695825.783593:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695825.783594:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695825.783598:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695825.783600:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695825.783601:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695825.783602:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695825.783605:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5800. 00000100:00000001:1.0:1787695825.783607:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:0.0:1787695825.783608:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787695825.783614:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:0.0:1787695825.783617:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000040:1.0:1787695825.783619:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000400:00000010:0.0:1787695825.783621:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b580. 00000020:00000001:1.0:1787695825.783623:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000010:0.0:1787695825.783625:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 00000020:00000040:1.0:1787695825.783627:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000100:00000200:0.0:1787695825.783630:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139528704, portal 10 00000020:00000001:1.0:1787695825.783632:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:0.0:1787695825.783633:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695825.783635:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929560192 : -128555779991424 : ffff8b1446665880) 00000100:00000040:0.0:1787695825.783641:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665880 x1874531139528704/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695850 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787695825.783644:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:0.0:1787695825.783652:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695825.783653:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:0.0:1787695825.783658:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787695825.783660:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695825.783664:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000010:0.0:1787695825.783667:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000001:1.0:1787695825.783669:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000200:0.0:1787695825.783671:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139528704, offset 0 00000020:00000040:1.0:1787695825.783675:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000400:00000200:0.0:1787695825.783678:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00080000:1.0:1787695825.783683:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:0.0:1787695825.783685:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695825.783692:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695825.783694:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 02000000:00000001:1.0:1787695825.783696:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1787695825.783700:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695825.783704:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 02000000:00000001:1.0:1787695825.783708:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1787695825.783709:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 02000000:00000001:1.0:1787695825.783714:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695825.783715:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:1.0:1787695825.783720:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:0.0:1787695825.783724:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16940:16940:256:4294967295] 192.168.201.140@tcp LPNI seq info [16939:16939:8:4294967295] 00000008:00100000:1.0:1787695825.783726:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787695825.783734:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695825.783735:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695825.783738:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695825.783745:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446664000. 00000400:00000200:0.0:1787695825.783748:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000020:00000040:1.0:1787695825.783750:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000800:00000200:0.0:1787695825.783755:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 02000000:00000001:1.0:1787695825.783757:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000010:0.0:1787695825.783760:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 02000000:00000001:1.0:1787695825.783761:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695825.783765:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:0.0:1787695825.783766:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:1.0:1787695825.783769:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:0.0:1787695825.783770:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 7 secs later) 02000000:00000001:1.0:1787695825.783775:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1787695825.783776:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695825.783781:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 7 secs later) 00000800:00000100:0.0:1787695825.783784:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 02000000:00000010:1.0:1787695825.783785:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545c00. 00000800:00000010:0.0:1787695825.783786:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:0.0:1787695825.783792:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1787695825.783794:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695825.783796:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695825.783801:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695825.783806:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00100000:1.0:1787695825.783806:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446664000 x1874531139528960/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695825.783811:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695825.783819:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446665880 x1874531139528704/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695850 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695825.783821:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695825.783826:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.783836:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695825.783837:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e60c2a2d] to pc [ptlrpcd_rcv+-1] req@ffff8b1446664000 x1874531139528960/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695825.783844:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665880 x1874531139528704/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695850 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695825.783859:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.783860:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.783861:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:1.0:1787695825.783871:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695825.783873:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695825.783879:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000100:00080000:1.0:1787695825.783880:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1787695825.783885:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1787695825.783887:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:1.0:1787695825.783888:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695825.783892:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298387398 00000100:00000001:1.0:1787695825.783894:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787695825.783897:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298397496 00000100:00000001:0.0:1787695825.783902:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298387398 00000100:00080000:1.0:1787695825.783903:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3713 00000100:00000001:0.0:1787695825.783907:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298397496 00000100:00000001:0.0:1787695825.783912:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1787695825.783913:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695825.783917:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695825.783921:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:0.0:1787695825.783922:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1787695825.783927:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787695825.783933:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695825/real 1787695825] req@ffff8b1446665880 x1874531139528704/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695825.783936:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695825.783942:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695825.783947:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000001:0.0:1787695825.783948:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695825.783951:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787695825.783953:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000010:0.0:1787695825.783954:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b580. 00000400:00000200:0.0:1787695825.783958:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000020:00000001:1.0:1787695825.783960:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000010:0.0:1787695825.783961:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000020:00000040:1.0:1787695825.783964:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000100:00000001:0.0:1787695825.783965:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:1.0:1787695825.783969:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000200:0.0:1787695825.783970:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665880 x1874531139528704/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1787695825.783974:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:1.0:1787695825.783982:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000200:0.0:1787695825.783984:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665880 x1874531139528704/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787695825.783988:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000001:0.0:1787695825.783996:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:1.0:1787695825.783998:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695825.783999:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.784002:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695825.784005:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695825.784007:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:0.0:1787695825.784011:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446665880 x1874531139528704/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695825.784014:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695825.784021:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695825.784025:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:1.0:1787695825.784027:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00000040:0.0:1787695825.784031:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665880 x1874531139528704/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695825.784034:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695825.784041:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695825.784046:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787695825.784049:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1787695825.784052:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695825.784053:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.784056:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695825.784061:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787695825.784061:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.784066:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695825.784067:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787695825.784069:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695825.784073:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695825.784079:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000008:00100000:1.0:1787695825.784079:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787695825.784084:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00000001:1.0:1787695825.784086:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695825.784090:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695825.784093:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665880 x1874531139528704/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1787695825.784096:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446666680. 00000020:00000040:1.0:1787695825.784102:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:1.0:1787695825.784108:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695825.784112:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695825.784113:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139528704:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695825.784115:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787695825.784120:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1787695825.784120:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695825.784123:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695825.784125:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446664a80 x1874531139528832/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1787695825.784132:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446547c00. 00000100:00000001:0.0:1787695825.784133:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695825.784135:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695825.784139:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695825.784141:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695825.784141:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695825.784146:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695825.784150:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446664a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139528832:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695825.784154:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:1.0:1787695825.784155:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446666680 x1874531139529088/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787695825.784156:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695825.784158:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695825.784162:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695825.784166:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5c00. 02000000:00000001:0.0:1787695825.784169:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695825.784172:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695825.784176:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695825.784181:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:1.0:1787695825.784186:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bd4d567] to pc [ptlrpcd_rcv+-1] req@ffff8b1446666680 x1874531139529088/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1787695825.784188:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b580. 00000400:00000010:0.0:1787695825.784194:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 00000100:00000200:0.0:1787695825.784199:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139528832, portal 10 00000100:00000001:1.0:1787695825.784202:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.784203:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695825.784207:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929556608 : -128555779995008 : ffff8b1446664a80) 00000100:00080000:1.0:1787695825.784208:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695825.784215:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787695825.784217:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:0.0:1787695825.784220:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446664a80 x1874531139528832/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695850 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695825.784221:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695825.784227:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695825.784234:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3713 00000100:00000001:0.0:1787695825.784237:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695825.784244:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695825.784244:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695825.784250:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000400:00000200:1.0:1787695825.784254:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000200:0.0:1787695825.784256:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139528832, offset 0 00000100:00080000:1.0:1787695825.784259:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1787695825.784267:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:1.0:1787695825.784267:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695825.784273:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1787695825.784278:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:1.0:1787695825.784278:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695825.784286:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695825.784291:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1787695825.784293:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:0.0:1787695825.784297:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000020:00000001:1.0:1787695825.784299:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:0.0:1787695825.784303:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000020:00000040:1.0:1787695825.784304:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000400:00000200:0.0:1787695825.784306:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000020:00000001:1.0:1787695825.784309:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:0.0:1787695825.784311:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16941:16941:256:4294967295] 192.168.201.140@tcp LPNI seq info [16940:16940:8:4294967295] 00000020:00000040:1.0:1787695825.784314:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000400:00000200:0.0:1787695825.784317:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:1.0:1787695825.784321:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000200:0.0:1787695825.784328:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000020:00000001:1.0:1787695825.784328:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000800:00000200:0.0:1787695825.784338:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000040:1.0:1787695825.784338:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000800:00000010:0.0:1787695825.784343:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000100:00000001:1.0:1787695825.784344:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.0:1787695825.784349:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:1.0:1787695825.784351:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000200:0.0:1787695825.784354:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 7 secs later) 00000100:00000001:1.0:1787695825.784357:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000200:0.1:1787695825.784361:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695825.784386:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 7 secs later) 00000800:00000100:0.0:1787695825.784392:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695825.784395:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000020:00000040:1.0:1787695825.784399:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000400:00000200:0.0:1787695825.784401:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:1.0:1787695825.784408:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:0.0:1787695825.784411:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1787695825.784415:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695825.784418:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 02000000:00000001:1.0:1787695825.784420:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695825.784425:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:0.0:1787695825.784441:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 02000000:00000001:1.0:1787695825.784443:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695825.784447:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787695825.784448:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000008:00100000:1.0:1787695825.784450:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787695825.784456:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695825.784457:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446664a80 x1874531139528832/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695850 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695825.784460:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695825.784469:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665180. 00000100:00000001:0.0:1787695825.784472:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000040:1.0:1787695825.784474:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000100:00000040:0.0:1787695825.784479:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446664a80 x1874531139528832/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695850 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695825.784480:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695825.784483:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695825.784488:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695825.784492:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787695825.784493:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.784496:0:2402:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1787695825.784496:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.784501:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695825.784504:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446547800. 00000100:00000040:0.0:1787695825.784505:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:0.0:1787695825.784511:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695825.784511:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.784513:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298386247 00000100:00000001:0.0:1787695825.784517:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298397496 00000100:00000001:0.0:1787695825.784519:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298386247 00000100:00100000:1.0:1787695825.784522:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446665180 x1874531139529216/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695825.784523:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298397496 00000100:00000001:0.0:1787695825.784527:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695825.784530:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.784533:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695825.784540:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695825/real 1787695825] req@ffff8b1446664a80 x1874531139528832/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695825.784546:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695825.784548:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695825.784551:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b580. 00000400:00000200:0.0:1787695825.784554:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:0.0:1787695825.784558:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:0.0:1787695825.784562:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695825.784566:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446664a80 x1874531139528832/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695825.784567:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695825.784573:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695825.784577:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446664a80 x1874531139528832/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695825.784583:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f923372e] to pc [ptlrpcd_rcv+-1] req@ffff8b1446665180 x1874531139529216/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.784604:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695825.784607:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.784610:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695825.784610:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.784612:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787695825.784616:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787695825.784619:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446664a80 x1874531139528832/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695825.784624:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787695825.784630:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695825.784630:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695825.784635:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:0.0:1787695825.784636:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446664a80 x1874531139528832/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695825.784641:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787695825.784646:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695825.784649:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3713 00000100:00000001:0.0:1787695825.784650:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.784653:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695825.784656:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.784658:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000400:00000200:1.0:1787695825.784660:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695825.784661:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695825.784666:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000400:00000200:1.0:1787695825.784670:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:0.0:1787695825.784673:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695825.784676:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:1.0:1787695825.784676:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695825.784686:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:0.0:1787695825.784687:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446664a80 x1874531139528832/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695825.784693:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695825.784700:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695825.784705:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00100000:0.0:1787695825.784706:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446664a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139528832:192.168.201.140@tcp:38:kworker.0 00000020:00000001:1.0:1787695825.784710:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695825.784714:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000100:00000001:0.0:1787695825.784715:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787695825.784717:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787695825.784720:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000040:1.0:1787695825.784721:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000100:00000040:0.0:1787695825.784728:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665880 x1874531139528704/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787695825.784728:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000020:00000001:1.0:1787695825.784733:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:1.0:1787695825.784743:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695825.784745:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1787695825.784748:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787695825.784752:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:1.0:1787695825.784754:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000010:0.0:1787695825.784755:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5800. 00000100:00000001:1.0:1787695825.784760:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:0.0:1787695825.784761:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695825.784762:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695825.784763:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695825.784765:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695825.784766:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545800. 02000000:00000001:0.0:1787695825.784768:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695825.784770:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695825.784773:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665880. 00000020:00000040:1.0:1787695825.784774:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00000001:0.0:1787695825.784775:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695825.784777:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.784781:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1787695825.784784:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000040:0.0:1787695825.784785:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446664a80 x1874531139528832/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695825.784793:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.784798:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695825.784800:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787695825.784801:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695825.784804:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5c00. 02000000:00000001:1.0:1787695825.784805:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695825.784808:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:1.0:1787695825.784811:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695825.784812:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695825.784815:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=18 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000100:00000001:1.0:1787695825.784817:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000020:00000001:0.0:1787695825.784820:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000008:00100000:1.0:1787695825.784821:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 02000000:00000010:0.0:1787695825.784824:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446546800. 00000008:00000001:1.0:1787695825.784828:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695825.784830:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1787695825.784832:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695825.784834:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695825.784838:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446666d80. 00000100:00000010:0.0:1787695825.784839:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446664a80. 00000100:00000001:0.0:1787695825.784846:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000020:00000040:1.0:1787695825.784846:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=20 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000001:0.0:1787695825.784849:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787695825.784852:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787695825.784853:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787695825.784855:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.784858:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695825.784859:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787695825.784861:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695825.784864:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787695825.784865:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695825.784868:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1787695825.784869:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.784870:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695825.784875:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446664000 x1874531139528960/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1787695825.784877:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446544000. 00000100:00000001:1.0:1787695825.784885:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.784886:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695825.784895:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695825.784895:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446666d80 x1874531139529344/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787695825.784901:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695825.784904:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695825.784909:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695825.784913:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1787695825.784917:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446664000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139528960:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695825.784918:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.784926:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1787695825.784929:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa82a36] to pc [ptlrpcd_rcv+-1] req@ffff8b1446666d80 x1874531139529344/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787695825.784930:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695825.784934:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695825.784938:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695825.784944:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5c00. 02000000:00000001:0.0:1787695825.784947:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695825.784947:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695825.784955:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1787695825.784957:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695825.784961:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b580. 00000100:00000040:1.0:1787695825.784963:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000010:0.0:1787695825.784965:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 00000100:00000040:1.0:1787695825.784969:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:0.0:1787695825.784970:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139528960, portal 4 00000100:00000001:0.0:1787695825.784974:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695825.784977:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929553920 : -128555779997696 : ffff8b1446664000) 00000100:00000040:1.0:1787695825.784979:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1787695825.784983:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446664000 x1874531139528960/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695850 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.784993:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695825.784996:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000040:1.0:1787695825.784996:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3723) 00000400:00000010:0.0:1787695825.785000:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695825.785003:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139528960, offset 0 00000400:00000200:0.0:1787695825.785006:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695825.785010:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695825.785013:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695825.785015:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695825.785017:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695825.785020:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695825.785023:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695825.785025:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16942:16942:256:4294967295] 192.168.201.140@tcp LPNI seq info [16941:16941:8:4294967295] 00000400:00000200:0.0:1787695825.785028:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695825.785033:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695825.785041:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695825.785043:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:0.0:1787695825.785046:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695825.785048:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 7 secs later) 00000800:00000200:0.1:1787695825.785051:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695825.785053:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 7 secs later) 00000800:00000100:0.0:1787695825.785055:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695825.785057:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:0.0:1787695825.785059:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695825.785063:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695825.785066:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695825.785068:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695825.785070:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695825.785075:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446664000 x1874531139528960/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695850 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785083:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695825.785086:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446664000 x1874531139528960/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695850 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785093:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.785095:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695825.785098:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695825.785102:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:0.0:1787695825.785106:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695825.785108:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298386277 00000100:00000001:0.0:1787695825.785111:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298397496 00000100:00000001:0.0:1787695825.785113:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298386277 00000100:00000001:0.0:1787695825.785115:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298397496 00000100:00000001:0.0:1787695825.785117:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695825.785119:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.785120:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695825.785124:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695825/real 1787695825] req@ffff8b1446664000 x1874531139528960/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695825.785134:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695825.785135:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695825.785137:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b580. 00000400:00000200:0.0:1787695825.785139:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:0.0:1787695825.785140:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:0.0:1787695825.785141:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695825.785144:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446664000 x1874531139528960/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695825.785149:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446664000 x1874531139528960/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785153:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695825.785154:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.785155:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695825.785156:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695825.785158:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446664000 x1874531139528960/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785162:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695825.785165:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446664000 x1874531139528960/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785170:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695825.785171:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.785172:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695825.785173:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.785174:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695825.785175:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695825.785177:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695825.785179:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695825.785181:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695825.785192:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446664000 x1874531139528960/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695825.785200:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446664000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139528960:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695825.785205:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695825.785208:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446666680 x1874531139529088/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785215:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695825.785217:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695825.785219:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695825.785221:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695825.785223:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695825.785226:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446666680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139529088:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695825.785229:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695825.785232:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695825.785233:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695825.785235:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695825.785239:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5800. 02000000:00000001:0.0:1787695825.785241:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695825.785245:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695825.785248:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b580. 00000400:00000010:0.0:1787695825.785251:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 00000100:00000200:0.0:1787695825.785255:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139529088, portal 4 00000100:00000001:0.0:1787695825.785257:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695825.785258:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929563776 : -128555779987840 : ffff8b1446666680) 00000100:00000040:0.0:1787695825.785264:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446666680 x1874531139529088/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695850 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785271:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695825.785274:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695825.785276:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695825.785279:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139529088, offset 0 00000400:00000200:0.0:1787695825.785290:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695825.785294:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695825.785297:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695825.785299:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695825.785301:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695825.785303:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695825.785305:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695825.785307:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16943:16943:256:4294967295] 192.168.201.140@tcp LPNI seq info [16942:16942:8:4294967295] 00000400:00000200:0.0:1787695825.785310:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695825.785314:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695825.785318:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695825.785320:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:0.0:1787695825.785323:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695825.785325:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 7 secs later) 00000800:00000200:0.1:1787695825.785327:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695825.785329:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 7 secs later) 00000800:00000100:0.0:1787695825.785332:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695825.785333:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:0.0:1787695825.785335:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695825.785338:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695825.785340:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695825.785341:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695825.785343:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695825.785346:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446666680 x1874531139529088/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695850 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785351:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695825.785353:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446666680 x1874531139529088/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695850 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785357:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.785358:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695825.785360:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695825.785381:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:0.0:1787695825.785385:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695825.785394:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298385950 00000100:00000001:0.0:1787695825.785397:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298397496 00000100:00000001:0.0:1787695825.785400:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298385950 00000100:00000001:0.0:1787695825.785402:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298397496 00000100:00000001:0.0:1787695825.785405:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695825.785408:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.785410:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695825.785416:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695825/real 1787695825] req@ffff8b1446666680 x1874531139529088/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695825.785423:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695825.785424:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695825.785425:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b580. 00000400:00000200:0.0:1787695825.785441:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:0.0:1787695825.785443:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:0.0:1787695825.785445:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695825.785449:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446666680 x1874531139529088/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695825.785458:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446666680 x1874531139529088/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785465:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695825.785467:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.785469:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695825.785470:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695825.785474:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446666680 x1874531139529088/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785482:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695825.785486:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446666680 x1874531139529088/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785491:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695825.785497:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.785498:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695825.785499:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.785501:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695825.785502:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695825.785504:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695825.785506:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695825.785508:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695825.785510:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446666680 x1874531139529088/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695825.785516:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446666680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139529088:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695825.785518:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695825.785520:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665180 x1874531139529216/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785524:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695825.785526:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695825.785527:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695825.785529:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695825.785530:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695825.785532:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139529216:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695825.785534:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695825.785535:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695825.785536:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695825.785537:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695825.785539:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f7400. 02000000:00000001:0.0:1787695825.785540:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695825.785543:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695825.785545:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b580. 00000400:00000010:0.0:1787695825.785547:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 00000100:00000200:0.0:1787695825.785549:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139529216, portal 10 00000100:00000001:0.0:1787695825.785554:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695825.785554:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929558400 : -128555779993216 : ffff8b1446665180) 00000100:00000040:0.0:1787695825.785559:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665180 x1874531139529216/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695850 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785563:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695825.785565:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695825.785566:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695825.785568:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139529216, offset 0 00000400:00000200:0.0:1787695825.785571:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695825.785573:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695825.785576:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695825.785578:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695825.785580:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695825.785581:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695825.785583:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695825.785585:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16944:16944:256:4294967295] 192.168.201.140@tcp LPNI seq info [16943:16943:8:4294967295] 00000400:00000200:0.0:1787695825.785588:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695825.785592:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695825.785595:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695825.785597:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:0.0:1787695825.785600:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695825.785601:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 7 secs later) 00000800:00000200:0.1:1787695825.785604:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695825.785606:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 7 secs later) 00000800:00000100:0.0:1787695825.785609:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695825.785610:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:0.0:1787695825.785612:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695825.785614:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695825.785616:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695825.785617:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695825.785619:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695825.785621:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446665180 x1874531139529216/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695850 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785641:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695825.785645:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665180 x1874531139529216/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695850 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785652:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.785653:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695825.785658:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695825.785661:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:0.0:1787695825.785664:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695825.785666:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298386136 00000100:00000001:0.0:1787695825.785669:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298397496 00000100:00000001:0.0:1787695825.785671:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298386136 00000100:00000001:0.0:1787695825.785674:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298397496 00000100:00000001:0.0:1787695825.785676:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695825.785678:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.785680:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695825.785685:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695825/real 1787695825] req@ffff8b1446665180 x1874531139529216/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695825.785694:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695825.785696:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695825.785697:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b580. 00000400:00000200:0.0:1787695825.785700:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:0.0:1787695825.785701:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:0.0:1787695825.785704:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695825.785707:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665180 x1874531139529216/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695825.785716:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665180 x1874531139529216/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785723:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695825.785730:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.785732:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695825.785734:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695825.785739:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446665180 x1874531139529216/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785745:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695825.785748:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665180 x1874531139529216/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785753:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695825.785754:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.785755:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695825.785756:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.785757:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695825.785758:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695825.785760:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695825.785762:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695825.785763:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695825.785765:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665180 x1874531139529216/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695825.785770:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139529216:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695825.785772:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.785774:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695825.785776:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446664000 x1874531139528960/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785780:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695825.785781:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695825.785783:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5c00. 02000000:00000001:0.0:1787695825.785785:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695825.785786:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695825.785791:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:0.0:1787695825.785793:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695825.785794:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545c00. 02000000:00000001:0.0:1787695825.785797:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695825.785798:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695825.785799:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446664000. 00000100:00000001:0.0:1787695825.785801:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695825.785802:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.785804:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695825.785808:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446666680 x1874531139529088/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785815:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695825.785817:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695825.785818:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5800. 02000000:00000001:0.0:1787695825.785820:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695825.785821:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695825.785823:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:0.0:1787695825.785825:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695825.785826:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446547c00. 02000000:00000001:0.0:1787695825.785829:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695825.785830:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695825.785832:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446666680. 00000100:00000001:0.0:1787695825.785835:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695825.785837:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.785838:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695825.785842:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665180 x1874531139529216/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785849:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695825.785850:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695825.785851:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f7400. 02000000:00000001:0.0:1787695825.785853:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695825.785854:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695825.785855:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695825.785856:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695825.785857:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446547800. 02000000:00000001:0.0:1787695825.785858:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695825.785864:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695825.785872:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665180. 00000100:00000001:0.0:1787695825.785874:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695825.785875:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.785877:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.785883:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695825.785885:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.785887:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695825.785889:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695825.785890:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695825.785894:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446666d80 x1874531139529344/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785901:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695825.785902:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695825.785905:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695825.785906:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695825.785909:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695825.785912:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446666d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139529344:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695825.785915:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695825.785918:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695825.785920:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695825.785922:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695825.785925:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f7400. 02000000:00000001:0.0:1787695825.785927:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695825.785931:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695825.785934:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b580. 00000400:00000010:0.0:1787695825.785937:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 00000100:00000200:0.0:1787695825.785939:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139529344, portal 10 00000100:00000001:0.0:1787695825.785941:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695825.785942:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929565568 : -128555779986048 : ffff8b1446666d80) 00000100:00000040:0.0:1787695825.785946:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446666d80 x1874531139529344/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695850 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.785951:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695825.785952:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695825.785958:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695825.785960:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139529344, offset 0 00000400:00000200:0.0:1787695825.785962:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695825.785965:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695825.785968:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695825.785970:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695825.785972:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695825.785973:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695825.785975:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695825.785977:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16945:16945:256:4294967295] 192.168.201.140@tcp LPNI seq info [16944:16944:8:4294967295] 00000400:00000200:0.0:1787695825.785980:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695825.785985:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695825.785987:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695825.785989:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:0.0:1787695825.785992:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695825.785994:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 7 secs later) 00000800:00000200:0.1:1787695825.785997:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695825.785999:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 7 secs later) 00000800:00000100:0.0:1787695825.786001:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695825.786002:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:0.0:1787695825.786004:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695825.786007:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695825.786009:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695825.786010:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695825.786012:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695825.786014:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446666d80 x1874531139529344/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695850 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.786019:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695825.786021:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446666d80 x1874531139529344/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695850 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.786026:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.786027:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695825.786033:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695825.786035:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:0.0:1787695825.786037:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695825.786039:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298386114 00000100:00000001:0.0:1787695825.786040:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298397496 00000100:00000001:0.0:1787695825.786042:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298386114 00000100:00000001:0.0:1787695825.786043:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298397496 00000100:00000001:0.0:1787695825.786045:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695825.786046:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.786047:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695825.786050:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695825/real 1787695825] req@ffff8b1446666d80 x1874531139529344/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695825.786055:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695825.786056:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695825.786057:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b580. 00000400:00000200:0.0:1787695825.786059:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:0.0:1787695825.786060:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:0.0:1787695825.786062:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695825.786065:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446666d80 x1874531139529344/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695825.786074:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446666d80 x1874531139529344/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.786080:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695825.786082:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.786084:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695825.786085:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695825.786089:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446666d80 x1874531139529344/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.786096:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695825.786100:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446666d80 x1874531139529344/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.786114:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695825.786117:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.786118:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695825.786119:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.786121:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695825.786122:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695825.786125:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695825.786129:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695825.786131:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695825.786135:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446666d80 x1874531139529344/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695825.786143:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446666d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139529344:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695825.786147:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.786150:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695825.786153:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446666d80 x1874531139529344/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695850 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695825.786160:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695825.786162:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695825.786164:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f7400. 02000000:00000001:0.0:1787695825.786167:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695825.786168:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695825.786170:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=19 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695825.786172:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695825.786174:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446544000. 02000000:00000001:0.0:1787695825.786177:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695825.786179:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695825.786180:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446666d80. 00000100:00000001:0.0:1787695825.786183:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695825.786185:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.786193:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695825.786197:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695825.786199:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695825.786201:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695825.786202:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695825.846487:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695825.846493:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695825.846510:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695825.846515:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695825.846517:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695825.846520:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695825.846522:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695825.846524:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695825.846526:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695826.294469:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695826.294482:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695826.294482:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695826.294486:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695826.294492:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695826.294494:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695826.294495:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695826.294503:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695826.294506:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695826.294508:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695826.294512:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695826.294513:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695826.294527:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695826.294533:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695826.486512:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695826.486518:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695826.486522:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695826.486523:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695826.486526:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695826.486529:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695826.486531:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695826.486536:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695826.486537:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695826.486538:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695826.486541:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695826.486542:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695826.486544:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695826.486546:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695826.486548:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695826.486549:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695826.486550:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695826.486551:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695826.486552:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695826.486553:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695826.486554:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695826.486555:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695826.486556:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695826.486557:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695826.486559:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695826.486560:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695826.486563:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695826.486564:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695826.486565:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695826.486566:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695826.486567:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695826.486568:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695826.486569:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695826.486569:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695826.486570:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695826.486572:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695826.486642:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695826.486646:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695826.486656:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787695826.806462:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695826.806478:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695826.806481:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695826.806487:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695826.806489:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695826.806494:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695826.806498:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695826.870465:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695826.870472:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695826.870474:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695826.870478:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695826.870481:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695826.870484:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695826.870485:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695826.870488:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695826.870489:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695827.318444:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695827.318470:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695827.318486:0:2403:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695827.318500:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695827.318500:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695827.318503:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695827.318508:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695827.318509:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695827.318510:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695827.318529:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695827.318530:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695827.318531:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695827.318543:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695827.318544:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695827.318544:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695827.318567:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695827.318567:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695827.318569:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695827.318580:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695827.318581:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695827.318594:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695827.510456:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695827.510478:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695827.510482:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695827.510483:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695827.510486:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695827.510487:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695827.510489:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695827.510490:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695827.510492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695827.510493:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695827.510495:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695827.510496:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695827.510547:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695827.510550:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695827.510552:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787695827.830424:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695827.830436:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695827.830438:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695827.830442:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695827.830444:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695827.830447:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695827.830450:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695827.894475:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695827.894482:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695827.894483:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695827.894488:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695827.894491:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695827.894500:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695827.894518:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695827.894520:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695827.894522:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695828.342489:0:2403:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695828.342489:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695828.342489:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695828.342501:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695828.342502:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695828.342502:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695828.342505:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695828.342507:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695828.342507:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695828.342514:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695828.342514:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695828.342514:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695828.342518:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695828.342519:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695828.342519:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695828.342526:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695828.342527:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695828.342528:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695828.342530:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695828.342532:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695828.342533:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695828.534459:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695828.534466:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695828.534473:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695828.534474:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695828.534478:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695828.534479:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695828.534483:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695828.534484:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695828.534487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695828.534488:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695828.534492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695828.534493:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695828.534542:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695828.534545:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695828.534547:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787695828.855176:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695828.855188:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695828.855189:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695828.855193:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695828.855196:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695828.855199:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695828.855202:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695828.918430:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695828.918437:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695828.918440:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695828.918445:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695828.918450:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695828.918453:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695828.918456:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695828.918460:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695828.918462:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695829.366427:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695829.366443:0:2405:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787695829.366445:0:2403:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000400:3.0:1787695829.366462:0:2405:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695744/real 1787695744] req@ffff8b14465ae300 x1874531139517184/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695829 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695829.366478:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695829.366481:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:3.0:1787695829.366483:0:2405:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1787695829.366485:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695829.366489:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1787695829.366492:0:2405:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787695829.366494:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695829.366499:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1787695829.366499:0:2405:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bd68. 00000400:00000200:3.0:1787695829.366507:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8000 00000400:00000010:3.0:1787695829.366510:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8000. 00000100:00000001:3.0:1787695829.366515:0:2405:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787695829.366520:0:2405:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ae300 x1874531139517184/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695829 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1787695829.366529:0:2405:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ae300 x1874531139517184/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695829 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695829.366536:0:2405:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787695829.366538:0:2405:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695829.366547:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695829.366549:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787695829.366554:0:2405:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b14465ae300 x1874531139517184/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695829 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695829.366561:0:2405:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:3.0:1787695829.366564:0:2405:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b144515c000 already not connected (conn 20, was 11): DISCONN 00000100:00000001:3.0:1787695829.366567:0:2405:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:3.0:1787695829.366568:0:2405:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695829.366574:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695829.366575:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695829.366577:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695829.366579:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695829.366579:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695829.366584:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695829.366588:0:2406:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787695829.366589:0:2405:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695829.366592:0:2405:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695829.366599:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ae300 x1874531139517184/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695829 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:2.0:1787695829.366605:0:2406:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695744/real 1787695744] req@ffff8b14465afb80 x1874531139517056/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695829 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695829.366624:0:2405:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:3.0:1787695829.366635:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695829.366639:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695829.366641:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:2.0:1787695829.366643:0:2406:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695829.366649:0:2406:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:3.0:1787695829.366649:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ae300 x1874531139517184/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695829 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1787695829.366656:0:2406:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14456d1aa8. 00000400:00000200:2.0:1787695829.366662:0:2406:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b145078a000 00000400:00000010:2.0:1787695829.366670:0:2406:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b145078a000. 00000100:00100000:3.0:1787695829.366673:0:2405:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:608c526c-eb6e-4a54-80a5-8969828241eb:2405:1874531139517184:192.168.201.140@tcp:400:kworker.0 00000100:00000001:2.0:1787695829.366676:0:2406:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787695829.366685:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1787695829.366686:0:2406:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465afb80 x1874531139517056/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695829 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695829.366692:0:2405:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695829.366700:0:2405:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ae300 x1874531139517184/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695829 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695829.366707:0:2405:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787695829.366711:0:2405:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1787695829.366716:0:2405:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1445fe5a00. 00000100:00000200:2.0:1787695829.366724:0:2406:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465afb80 x1874531139517056/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695829 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695829.366739:0:2406:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695829.366741:0:2406:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695829.366743:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695829.366744:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695829.366749:0:2406:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b14465afb80 x1874531139517056/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695829 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695829.366749:0:2405:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787695829.366757:0:2405:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:2.0:1787695829.366759:0:2406:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:2.0:1787695829.366765:0:2406:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b1458081800 already not connected (conn 24, was 15): DISCONN 00000020:00000040:3.0:1787695829.366765:0:2405:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=18 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000001:2.0:1787695829.366776:0:2406:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000020:00000001:3.0:1787695829.366776:0:2405:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:2.0:1787695829.366785:0:2406:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1787695829.366792:0:2405:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b144a733e00. 00000100:00000001:2.0:1787695829.366794:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695829.366797:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=6 : 6 : 6) 02000000:00000001:3.0:1787695829.366798:0:2405:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1787695829.366801:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787695829.366802:0:2405:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787695829.366805:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000010:3.0:1787695829.366807:0:2405:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ae300. 00000100:00000001:2.0:1787695829.366809:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695829.366816:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695829.366818:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695829.366820:0:2405:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000040:2.0:1787695829.366825:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465afb80 x1874531139517056/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695829 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695829.366825:0:2405:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695829.366834:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695829.366836:0:2406:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:2.0:1787695829.366845:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695829.366848:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695829.366849:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695829.366852:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465afb80 x1874531139517056/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695829 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695829.366859:0:2406:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2406:1874531139517056:192.168.201.140@tcp:400:kworker.0 00000100:00000001:2.0:1787695829.366863:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695829.366872:0:2406:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695829.366876:0:2406:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465afb80 x1874531139517056/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695829 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695829.366881:0:2406:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695829.366884:0:2406:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695829.366887:0:2406:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b14474d5e00. 02000000:00000001:2.0:1787695829.366891:0:2406:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695829.366893:0:2406:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695829.366894:0:2406:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=17 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:2.0:1787695829.366896:0:2406:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695829.366899:0:2406:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b144a733a00. 02000000:00000001:2.0:1787695829.366902:0:2406:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695829.366903:0:2406:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695829.366905:0:2406:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465afb80. 00000100:00000001:2.0:1787695829.366908:0:2406:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695829.366909:0:2406:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695829.366912:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695829.558557:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695829.558569:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695829.558577:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695829.558579:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695829.558584:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695829.558586:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695829.558591:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695829.558592:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695829.558596:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695829.558622:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695829.558626:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695829.558628:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695829.558686:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695829.558690:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695829.558693:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787695829.878444:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695829.878458:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695829.878461:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695829.878467:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695829.878469:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695829.878474:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695829.878478:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695829.942477:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695829.942486:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695829.942488:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695829.942494:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695829.942498:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695829.942502:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695829.942504:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695829.942508:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695829.942510:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.390426:0:2403:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695830.390432:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695830.390433:0:2403:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787695830.390442:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695830.390445:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000400:0.0:1787695830.390449:0:2403:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695745/real 1787695745] req@ffff8b1446665c00 x1874531139517312/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695830 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695830.390451:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695830.390454:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695830.390459:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695830.390463:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787695830.390466:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695830.390469:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695830.390472:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b318. 00000400:00000200:0.0:1787695830.390477:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96088 00000400:00000010:0.0:1787695830.390480:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96088. 00000100:00000001:0.0:1787695830.390483:0:2403:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695830.390486:0:2403:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665c00 x1874531139517312/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695830 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695830.390493:0:2403:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665c00 x1874531139517312/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695830 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.390499:0:2403:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695830.390501:0:2403:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.390504:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695830.390506:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695830.390510:0:2403:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446665c00 x1874531139517312/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695830 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.390516:0:2403:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:0.0:1787695830.390518:0:2403:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b1458081800 already not connected (conn 24, was 15): DISCONN 00000100:00000001:0.0:1787695830.390521:0:2403:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:0.0:1787695830.390522:0:2403:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.390527:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695830.390528:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=14 : 14 : e) 00000100:00000001:0.0:1787695830.390530:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695830.390546:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695830.390549:0:2403:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.390551:0:2403:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695830.390552:0:2403:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695830.390556:0:2403:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665c00 x1874531139517312/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695830 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.390562:0:2403:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:0.0:1787695830.390564:0:2403:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.390566:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695830.390567:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695830.390570:0:2403:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665c00 x1874531139517312/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695830 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695830.390576:0:2403:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2403:1874531139517312:192.168.201.140@tcp:400:kworker.0 00000100:00000001:0.0:1787695830.390579:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.390581:0:2403:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695830.390583:0:2403:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665c00 x1874531139517312/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695830 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.390588:0:2403:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695830.390590:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695830.390593:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b14515c3000. 02000000:00000001:0.0:1787695830.390597:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695830.390599:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695830.390600:0:2403:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=16 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695830.390601:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695830.390603:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1445678c00. 02000000:00000001:0.0:1787695830.390606:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695830.390608:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695830.390609:0:2403:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665c00. 00000100:00000001:0.0:1787695830.390612:0:2403:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695830.390613:0:2403:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.390615:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695830.582479:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695830.582486:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695830.582491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695830.582492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695830.582494:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695830.582496:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695830.582499:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695830.582500:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695830.582502:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695830.582503:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695830.582505:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695830.582506:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695830.582579:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695830.582606:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695830.582610:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787695830.902452:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695830.902463:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787695830.902466:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787695830.902469:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:3.0:1787695830.902470:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695830.902476:0:2404:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787695830.902477:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695830.902486:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3718 00000100:00000400:1.0:1787695830.902491:0:2404:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695745/real 1787695745] req@ffff8b1446666a00 x1874531139517440/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695830 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695830.902501:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000001:1.0:1787695830.902508:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695830.902510:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695830.902513:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547022c0. 00000400:00000200:1.0:1787695830.902518:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46e58 00000400:00000010:1.0:1787695830.902520:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46e58. 00000400:00000200:3.0:1787695830.902523:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695830.902526:0:2404:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:3.0:1787695830.902529:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000200:1.0:1787695830.902532:0:2404:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446666a00 x1874531139517440/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695830 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695830.902537:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000200:1.0:1787695830.902544:0:2404:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446666a00 x1874531139517440/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695830 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695830.902546:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695830.902553:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695830.902556:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695830.902563:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695830.902567:0:2404:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:3.0:1787695830.902567:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000100:00000001:1.0:1787695830.902571:0:2404:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787695830.902572:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695830.902574:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695830.902577:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1787695830.902579:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787695830.902585:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787695830.902588:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00100000:1.0:1787695830.902591:0:2404:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446666a00 x1874531139517440/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695830 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695830.902593:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695830.902599:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695830.902602:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695830.902605:0:2404:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:1.0:1787695830.902608:0:2404:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b144515c000 already not connected (conn 20, was 11): DISCONN 00000100:00000001:3.0:1787695830.902614:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695830.902615:0:2404:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:1.0:1787695830.902617:0:2404:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695830.902625:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:3.0:1787695830.902625:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00000001:1.0:1787695830.902629:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=14 : 14 : e) 00000100:00080000:3.0:1787695830.902633:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695830.902634:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695830.902637:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695830.902640:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695830.902641:0:2404:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695830.902647:0:2404:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695830.902648:0:2404:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695830.902648:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695830.902655:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1787695830.902656:0:2404:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446666a00 x1874531139517440/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695830 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695830.902664:0:2404:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:1.0:1787695830.902666:0:2404:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787695830.902668:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695830.902669:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695830.902671:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695830.902676:0:2404:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446666a00 x1874531139517440/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695830 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695830.902677:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787695830.902681:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695830.902689:0:2404:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446666a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:608c526c-eb6e-4a54-80a5-8969828241eb:2404:1874531139517440:192.168.201.140@tcp:400:kworker.0 00000100:00000010:3.0:1787695830.902690:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5180. 00000100:00000001:1.0:1787695830.902696:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695830.902696:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 00000100:00000001:1.0:1787695830.902701:0:2404:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1787695830.902703:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000040:1.0:1787695830.902704:0:2404:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446666a00 x1874531139517440/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695830 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695830.902705:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695830.902711:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695830.902714:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695830.902717:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695830.902724:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674800. 00000100:00000001:1.0:1787695830.902730:0:2404:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695830.902735:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1787695830.902737:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695830.902738:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b144667d000. 02000000:00000001:1.0:1787695830.902744:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:3.0:1787695830.902747:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5180 x1874531139529472/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:1.0:1787695830.902748:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695830.902752:0:2404:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=17 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695830.902756:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695830.902759:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1445678e00. 02000000:00000001:1.0:1787695830.902766:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1787695830.902767:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787695830.902769:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695830.902773:0:2404:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446666a00. 00000100:00000001:3.0:1787695830.902774:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695830.902778:0:2404:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695830.902783:0:2404:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695830.902784:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a115abf4] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5180 x1874531139529472/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695830.902789:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695830.902810:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695830.902814:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695830.902818:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695830.902820:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695830.902822:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695830.902826:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695830.902829:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3718 00000100:00000001:0.0:1787695830.902834:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1787695830.902835:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695830.902839:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695830.902842:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000400:00000200:3.0:1787695830.902842:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695830.902846:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1787695830.902851:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5180 x1874531139529472/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695830.902852:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695830.902857:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695830.902861:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695830.902866:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695830.902867:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695830.902870:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695830.902870:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695830.902873:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 02000000:00000001:0.0:1787695830.902875:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1787695830.902876:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1787695830.902879:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1787695830.902880:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 02000000:00000001:0.0:1787695830.902884:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695830.902885:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:3.0:1787695830.902890:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00100000:0.0:1787695830.902891:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139529472:192.168.201.140@tcp:250:kworker.0 00000100:00000040:3.0:1787695830.902895:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695830.902897:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787695830.902898:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787695830.902901:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695830.902903:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695830.902906:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1787695830.902908:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695830.902910:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000010:0.0:1787695830.902912:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5800. 00000020:00000040:3.0:1787695830.902913:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 02000000:00000001:0.0:1787695830.902918:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695830.902919:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695830.902923:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695830.902925:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1787695830.902929:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787695830.902929:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695830.902933:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695830.902934:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:0.0:1787695830.902935:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b318. 00000008:00100000:3.0:1787695830.902938:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695830.902941:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695830.902943:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695830.902944:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96088. 00000100:00000010:3.0:1787695830.902947:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4380. 00000020:00000040:3.0:1787695830.902950:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000100:00000200:0.0:1787695830.902952:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139529472, portal 25 02000000:00000001:3.0:1787695830.902953:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695830.902956:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.902957:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787695830.902958:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787695830.902961:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930148224 : -128555779403392 : ffff8b14466f5180) 02000000:00000001:3.0:1787695830.902961:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695830.902964:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695830.902970:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 00000100:00000001:3.0:1787695830.902974:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695830.902975:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5180 x1874531139529472/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695855 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695830.902982:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4380 x1874531139529600/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695830.902996:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787695830.902998:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:0.0:1787695830.903004:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695830.903009:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 00000100:00000001:3.0:1787695830.903010:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695830.903015:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139529472, offset 0 00000100:00000040:3.0:1787695830.903016:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d2982c15] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4380 x1874531139529600/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695830.903025:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695830.903028:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695830.903031:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695830.903034:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695830.903035:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1787695830.903040:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:3.0:1787695830.903046:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1787695830.903049:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:3.0:1787695830.903052:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695830.903057:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3718 00000400:00000200:0.0:1787695830.903063:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1787695830.903067:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695830.903070:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695830.903077:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1787695830.903078:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000200:0.0:1787695830.903081:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00080000:3.0:1787695830.903084:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1787695830.903086:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16946:16946:256:4294967295] 192.168.201.140@tcp LPNI seq info [16945:16945:8:4294967295] 00000100:00000040:3.0:1787695830.903092:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695830.903093:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695830.903098:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695830.903100:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695830.903102:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695830.903104:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695830.903105:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:3.0:1787695830.903107:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695830.903111:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000400:00000200:0.0:1787695830.903114:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000020:00000001:3.0:1787695830.903116:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000800:00000200:0.0:1787695830.903121:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000020:00000001:3.0:1787695830.903122:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000800:00000010:0.0:1787695830.903126:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000100:00000040:3.0:1787695830.903130:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695830.903135:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.0:1787695830.903136:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695830.903141:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 2 secs later) 00000100:00000040:3.0:1787695830.903141:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695830.903147:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000200:0.1:1787695830.903148:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000020:00000040:3.0:1787695830.903151:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:3.0:1787695830.903156:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000800:00000200:0.1:1787695830.903159:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 2 secs later) 00000100:00000001:3.0:1787695830.903160:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695830.903165:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:0.0:1787695830.903168:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695830.903169:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:0.0:1787695830.903171:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 02000000:00000001:3.0:1787695830.903174:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695830.903177:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1787695830.903177:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695830.903180:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695830.903183:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695830.903186:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695830.903188:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695830.903193:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:0.0:1787695830.903194:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:0.0:1787695830.903197:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695830.903201:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5180 x1874531139529472/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695855 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1787695830.903204:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6d80. 00000020:00000040:3.0:1787695830.903208:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=17 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695830.903213:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695830.903215:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.903216:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787695830.903218:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:0.0:1787695830.903222:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5180 x1874531139529472/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695855 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695830.903222:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695830.903225:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695830.903231:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 00000100:00000001:0.0:1787695830.903233:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695830.903235:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.903236:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695830.903242:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695830.903243:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6d80 x1874531139529728/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1787695830.903249:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695830.903254:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298402616 00000100:00000001:3.0:1787695830.903257:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695830.903260:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.903261:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298397496 00000100:00000040:3.0:1787695830.903265:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c3be57b] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6d80 x1874531139529728/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.903266:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298402616 00000100:00000001:3.0:1787695830.903272:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695830.903275:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695830.903277:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298397496 00000100:00080000:3.0:1787695830.903278:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787695830.903282:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695830.903282:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695830.903284:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695830.903287:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695830.903289:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3718 00000100:00000001:0.0:1787695830.903291:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1787695830.903295:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695830.903298:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000400:00000200:3.0:1787695830.903299:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695830.903303:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787695830.903305:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695830/real 1787695830] req@ffff8b14466f5180 x1874531139529472/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695830.903310:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695830.903317:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:0.0:1787695830.903319:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695830.903322:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:3.0:1787695830.903323:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000010:0.0:1787695830.903325:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b318. 00000100:00000001:3.0:1787695830.903330:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:0.0:1787695830.903332:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96088 00000020:00000001:3.0:1787695830.903335:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000010:0.0:1787695830.903336:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96088. 00000020:00000040:3.0:1787695830.903360:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:3.0:1787695830.903362:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695830.903382:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:3.0:1787695830.903384:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:3.0:1787695830.903386:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:3.0:1787695830.903389:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695830.903391:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695830.903393:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695830.903395:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695830.903397:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:3.0:1787695830.903400:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695830.903402:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695830.903404:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695830.903406:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695830.903409:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695830.903411:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787695830.903412:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000008:00100000:3.0:1787695830.903414:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787695830.903417:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695830.903419:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5180 x1874531139529472/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695830.903420:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695830.903426:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5f80. 00000020:00000040:3.0:1787695830.903430:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000100:00000200:0.0:1787695830.903433:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5180 x1874531139529472/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695830.903435:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695830.903437:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695830.903441:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695830.903443:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695830.903446:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.903447:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695830.903449:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1787695830.903450:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 00000100:00000001:0.0:1787695830.903451:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695830.903455:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695830.903456:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695830.903459:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5180 x1874531139529472/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695830.903464:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5f80 x1874531139529856/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695830.903473:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695830.903478:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:0.0:1787695830.903481:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5180 x1874531139529472/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695830.903481:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.903491:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:3.0:1787695830.903494:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006e7bde76] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5f80 x1874531139529856/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.903495:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.903499:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695830.903501:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.903505:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787695830.903508:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695830.903510:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695830.903514:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787695830.903515:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1787695830.903519:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787695830.903519:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695830.903521:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695830.903527:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787695830.903529:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00080000:3.0:1787695830.903532:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3718 00000100:00000001:0.0:1787695830.903533:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000400:00000200:3.0:1787695830.903537:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:0.0:1787695830.903541:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5180 x1874531139529472/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695830.903545:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695830.903549:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695830.903554:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695830.903558:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:0.0:1787695830.903561:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139529472:192.168.201.140@tcp:250:kworker.0 00000100:00000040:3.0:1787695830.903563:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695830.903568:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695830.903570:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787695830.903570:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695830.903573:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000100:00000001:0.0:1787695830.903574:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1787695830.903575:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695830.903579:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000100:00000040:0.0:1787695830.903580:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5180 x1874531139529472/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695830.903582:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:3.0:1787695830.903587:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:3.0:1787695830.903594:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695830.903595:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1787695830.903597:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787695830.903598:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:3.0:1787695830.903601:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000010:0.0:1787695830.903604:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5800. 00000100:00000001:3.0:1787695830.903604:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695830.903610:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 02000000:00000001:0.0:1787695830.903612:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:3.0:1787695830.903613:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000020:00000001:0.0:1787695830.903617:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:3.0:1787695830.903617:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1787695830.903620:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787695830.903626:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:0.0:1787695830.903627:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787695830.903629:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:0.0:1787695830.903631:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674800. 02000000:00000001:3.0:1787695830.903632:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695830.903636:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787695830.903639:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000008:00100000:3.0:1787695830.903639:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 02000000:00000001:0.0:1787695830.903643:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000008:00000001:3.0:1787695830.903643:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1787695830.903647:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5180. 00000100:00000001:3.0:1787695830.903647:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695830.903651:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5500. 00000100:00000001:0.0:1787695830.903654:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000020:00000040:3.0:1787695830.903654:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 00000100:00000001:0.0:1787695830.903656:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787695830.903658:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787695830.903659:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787695830.903661:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695830.903663:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787695830.903664:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1787695830.903665:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787695830.903668:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695830.903669:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.903672:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695830.903674:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000010:3.0:1787695830.903674:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675400. 00000100:00000001:0.0:1787695830.903678:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787695830.903679:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695830.903683:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4380 x1874531139529600/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695830.903686:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5500 x1874531139529984/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695830.903706:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695830.903707:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695830.903710:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a777d68] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5500 x1874531139529984/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.903732:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695830.903735:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695830.903735:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695830.903740:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1787695830.903741:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695830.903744:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1787695830.903745:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 02000000:00000001:0.0:1787695830.903748:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695830.903748:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695830.903751:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695830.903756:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695830.903758:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3718 00000400:00000200:3.0:1787695830.903761:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00100000:0.0:1787695830.903763:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139529600:192.168.201.140@tcp:38:kworker.0 00000400:00000200:3.0:1787695830.903764:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:0.0:1787695830.903769:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1787695830.903771:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 02000000:00000001:0.0:1787695830.903772:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695830.903774:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695830.903775:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695830.903778:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5800. 02000000:00000001:0.0:1787695830.903783:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695830.903786:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695830.903788:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695830.903793:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b318. 00000100:00000001:3.0:1787695830.903793:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:0.0:1787695830.903799:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96088. 00000100:00000040:3.0:1787695830.903800:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000200:0.0:1787695830.903802:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139529600, portal 10 00000100:00000001:3.0:1787695830.903805:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695830.903806:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695830.903808:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930144640 : -128555779406976 : ffff8b14466f4380) 00000020:00000001:3.0:1787695830.903812:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695830.903816:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000100:00000040:0.0:1787695830.903817:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4380 x1874531139529600/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695855 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695830.903818:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695830.903822:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000020:00000001:3.0:1787695830.903826:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:0.0:1787695830.903830:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1787695830.903831:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000040:0.0:1787695830.903836:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695830.903837:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000010:0.0:1787695830.903839:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 00000100:00000001:3.0:1787695830.903840:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:0.0:1787695830.903844:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139529600, offset 0 00000100:00000040:3.0:1787695830.903845:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695830.903849:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:0.0:1787695830.903853:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000020:00000040:3.0:1787695830.903853:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:3.0:1787695830.903858:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:0.0:1787695830.903859:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695830.903862:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695830.903865:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1787695830.903866:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 02000000:00000001:3.0:1787695830.903868:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1787695830.903870:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1787695830.903873:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695830.903875:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:3.0:1787695830.903876:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:0.0:1787695830.903880:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000008:00100000:3.0:1787695830.903881:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1787695830.903884:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000008:00000001:3.0:1787695830.903885:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695830.903887:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695830.903890:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4700. 00000400:00000200:0.0:1787695830.903894:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16947:16947:256:4294967295] 192.168.201.140@tcp LPNI seq info [16946:16946:8:4294967295] 00000020:00000040:3.0:1787695830.903894:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:3.0:1787695830.903898:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1787695830.903901:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787695830.903901:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695830.903904:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695830.903906:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695830.903910:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695830.903914:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 00000400:00000200:0.0:1787695830.903915:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695830.903923:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695830.903924:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1787695830.903929:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000100:00100000:3.0:1787695830.903931:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4700 x1874531139530112/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.0:1787695830.903940:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787695830.903941:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695830.903946:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1787695830.903947:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 2 secs later) 00000100:00000040:3.0:1787695830.903951:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008818c2cc] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4700 x1874531139530112/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.1:1787695830.903958:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787695830.903963:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1787695830.903964:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 2 secs later) 00000100:00080000:3.0:1787695830.903967:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787695830.903973:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00080000:3.0:1787695830.903973:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000010:0.0:1787695830.903978:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000100:00000001:3.0:1787695830.903978:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695830.903979:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695830.903984:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:0.0:1787695830.903985:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:3.0:1787695830.903987:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3718 00000400:00000200:3.0:1787695830.903994:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695830.903997:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1787695830.904000:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695830.904003:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1787695830.904004:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000100:00000040:3.0:1787695830.904006:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000010:0.0:1787695830.904008:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:3.0:1787695830.904009:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695830.904014:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695830.904015:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787695830.904016:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695830.904020:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695830.904021:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000020:00000001:3.0:1787695830.904023:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000200:0.0:1787695830.904024:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4380 x1874531139529600/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695855 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1787695830.904025:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:3.0:1787695830.904030:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000020:00000001:3.0:1787695830.904034:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:0.0:1787695830.904039:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695830.904042:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:0.0:1787695830.904044:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4380 x1874531139529600/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695855 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695830.904048:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695830.904052:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.904053:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695830.904056:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695830.904061:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000100:00000040:3.0:1787695830.904061:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695830.904068:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695830.904068:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695830.904072:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298392518 00000020:00000040:3.0:1787695830.904074:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00000001:0.0:1787695830.904075:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298402616 00000100:00000001:0.0:1787695830.904080:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298392518 00000100:00000001:0.0:1787695830.904083:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298402616 00000100:00080000:3.0:1787695830.904083:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695830.904088:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695830.904089:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695830.904091:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695830.904094:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695830.904096:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.904098:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695830.904099:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787695830.904102:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000008:00100000:3.0:1787695830.904104:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787695830.904108:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695830/real 1787695830] req@ffff8b14466f4380 x1874531139529600/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1787695830.904110:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695830.904115:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695830.904122:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4000. 00000400:00000001:0.0:1787695830.904123:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695830.904125:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695830.904128:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b318. 00000020:00000040:3.0:1787695830.904129:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=18 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000400:00000200:0.0:1787695830.904130:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96088 00000400:00000010:0.0:1787695830.904133:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96088. 02000000:00000001:3.0:1787695830.904135:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787695830.904136:0:2402:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1787695830.904140:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695830.904141:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4380 x1874531139529600/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695830.904146:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695830.904149:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:0.0:1787695830.904152:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4380 x1874531139529600/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695830.904152:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695830.904158:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676000. 00000100:00000001:0.0:1787695830.904160:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695830.904162:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695830.904163:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.904164:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695830.904167:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787695830.904173:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4000 x1874531139530240/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787695830.904174:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4380 x1874531139529600/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904202:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695830.904203:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695830.904207:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695830.904208:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4380 x1874531139529600/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695830.904213:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5a8f356] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4000 x1874531139530240/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904216:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695830.904221:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.904224:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695830.904226:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.904230:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695830.904233:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787695830.904233:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695830.904238:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000040:3.0:1787695830.904238:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787695830.904242:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695830.904246:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787695830.904249:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695830.904251:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1787695830.904257:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:0.0:1787695830.904258:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4380 x1874531139529600/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695830.904264:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3728) 00000100:00100000:0.0:1787695830.904270:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139529600:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695830.904274:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695830.904276:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6d80 x1874531139529728/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904281:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695830.904282:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695830.904284:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695830.904285:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695830.904286:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695830.904288:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139529728:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695830.904290:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695830.904291:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695830.904292:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695830.904293:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695830.904296:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5c00. 02000000:00000001:0.0:1787695830.904297:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695830.904300:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695830.904301:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b318. 00000400:00000010:0.0:1787695830.904304:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96088. 00000100:00000200:0.0:1787695830.904306:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139529728, portal 10 00000100:00000001:0.0:1787695830.904307:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695830.904308:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930155392 : -128555779396224 : ffff8b14466f6d80) 00000100:00000040:0.0:1787695830.904313:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6d80 x1874531139529728/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695855 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904317:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695830.904322:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695830.904324:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 00000100:00000200:0.0:1787695830.904326:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139529728, offset 0 00000400:00000200:0.0:1787695830.904329:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695830.904333:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695830.904337:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695830.904352:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695830.904355:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695830.904358:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695830.904360:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695830.904382:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16948:16948:256:4294967295] 192.168.201.140@tcp LPNI seq info [16947:16947:8:4294967295] 00000400:00000200:0.0:1787695830.904387:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695830.904393:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695830.904396:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695830.904400:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:0.0:1787695830.904403:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695830.904405:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 2 secs later) 00000800:00000200:0.1:1787695830.904408:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695830.904409:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 2 secs later) 00000800:00000100:0.0:1787695830.904412:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695830.904414:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:0.0:1787695830.904416:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695830.904418:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695830.904420:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:0.0:1787695830.904422:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:0.0:1787695830.904423:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695830.904426:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6d80 x1874531139529728/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695855 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904432:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695830.904434:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6d80 x1874531139529728/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695855 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904439:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.904440:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695830.904447:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695830.904449:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:0.0:1787695830.904451:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695830.904453:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298391367 00000100:00000001:0.0:1787695830.904454:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298402616 00000100:00000001:0.0:1787695830.904456:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298391367 00000100:00000001:0.0:1787695830.904457:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298402616 00000100:00000001:0.0:1787695830.904459:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695830.904460:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.904462:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695830.904464:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695830/real 1787695830] req@ffff8b14466f6d80 x1874531139529728/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695830.904469:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695830.904470:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695830.904471:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b318. 00000400:00000200:0.0:1787695830.904473:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96088 00000400:00000010:0.0:1787695830.904474:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96088. 00000100:00000001:0.0:1787695830.904475:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695830.904477:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6d80 x1874531139529728/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695830.904483:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6d80 x1874531139529728/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904486:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695830.904487:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.904488:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695830.904489:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695830.904491:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6d80 x1874531139529728/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904495:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695830.904498:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6d80 x1874531139529728/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904506:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695830.904507:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.904509:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695830.904510:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.904511:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695830.904512:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695830.904513:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695830.904516:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695830.904517:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695830.904520:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6d80 x1874531139529728/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695830.904525:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139529728:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695830.904527:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695830.904529:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5f80 x1874531139529856/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904533:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695830.904534:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695830.904536:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695830.904537:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695830.904538:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695830.904540:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139529856:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695830.904542:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695830.904543:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695830.904544:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695830.904545:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695830.904547:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5400. 02000000:00000001:0.0:1787695830.904549:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695830.904551:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695830.904556:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b318. 00000400:00000010:0.0:1787695830.904558:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96088. 00000100:00000200:0.0:1787695830.904561:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139529856, portal 4 00000100:00000001:0.0:1787695830.904562:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695830.904563:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930151808 : -128555779399808 : ffff8b14466f5f80) 00000100:00000040:0.0:1787695830.904568:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5f80 x1874531139529856/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695855 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904572:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695830.904573:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695830.904575:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 00000100:00000200:0.0:1787695830.904576:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139529856, offset 0 00000400:00000200:0.0:1787695830.904579:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695830.904581:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695830.904584:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695830.904586:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695830.904587:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695830.904589:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695830.904591:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695830.904593:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16949:16949:256:4294967295] 192.168.201.140@tcp LPNI seq info [16948:16948:8:4294967295] 00000400:00000200:0.0:1787695830.904595:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695830.904601:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695830.904606:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695830.904609:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:0.0:1787695830.904613:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695830.904616:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 2 secs later) 00000800:00000200:0.1:1787695830.904620:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695830.904623:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 2 secs later) 00000800:00000100:0.0:1787695830.904632:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695830.904634:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:0.0:1787695830.904637:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695830.904641:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695830.904644:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:0.0:1787695830.904646:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:0.0:1787695830.904655:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695830.904659:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5f80 x1874531139529856/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695855 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904664:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695830.904666:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5f80 x1874531139529856/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695855 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904670:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.904671:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695830.904674:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695830.904676:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:0.0:1787695830.904678:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695830.904680:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298391397 00000100:00000001:0.0:1787695830.904682:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298402616 00000100:00000001:0.0:1787695830.904683:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298391397 00000100:00000001:0.0:1787695830.904684:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298402616 00000100:00000001:0.0:1787695830.904686:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695830.904687:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.904688:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695830.904691:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695830/real 1787695830] req@ffff8b14466f5f80 x1874531139529856/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695830.904695:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695830.904696:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695830.904697:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b318. 00000400:00000200:0.0:1787695830.904699:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96088 00000400:00000010:0.0:1787695830.904700:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96088. 00000100:00000001:0.0:1787695830.904701:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695830.904704:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5f80 x1874531139529856/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695830.904709:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5f80 x1874531139529856/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904720:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695830.904722:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.904723:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695830.904724:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695830.904727:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5f80 x1874531139529856/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904731:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695830.904733:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5f80 x1874531139529856/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904738:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695830.904739:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.904740:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695830.904741:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.904742:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695830.904743:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695830.904745:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695830.904747:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695830.904748:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695830.904750:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5f80 x1874531139529856/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695830.904755:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139529856:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695830.904757:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.904758:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695830.904761:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4380 x1874531139529600/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904765:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695830.904766:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695830.904770:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5800. 02000000:00000001:0.0:1787695830.904772:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695830.904773:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695830.904775:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695830.904776:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695830.904777:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:0.0:1787695830.904779:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695830.904780:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695830.904781:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4380. 00000100:00000001:0.0:1787695830.904783:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695830.904784:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.904785:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695830.904787:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6d80 x1874531139529728/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904791:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695830.904792:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695830.904793:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5c00. 02000000:00000001:0.0:1787695830.904794:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695830.904795:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695830.904796:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=16 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695830.904797:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695830.904798:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:0.0:1787695830.904801:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695830.904802:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695830.904803:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6d80. 00000100:00000001:0.0:1787695830.904804:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695830.904805:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.904806:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695830.904808:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5f80 x1874531139529856/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904811:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695830.904812:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695830.904813:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5400. 02000000:00000001:0.0:1787695830.904814:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695830.904815:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695830.904815:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:0.0:1787695830.904819:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695830.904820:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:0.0:1787695830.904823:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695830.904823:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695830.904824:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5f80. 00000100:00000001:0.0:1787695830.904826:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695830.904826:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.904828:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.904830:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695830.904831:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.904832:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695830.904833:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695830.904834:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695830.904837:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5500 x1874531139529984/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904841:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695830.904842:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695830.904843:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695830.904844:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695830.904846:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695830.904848:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139529984:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695830.904849:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695830.904851:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695830.904852:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695830.904853:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695830.904856:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5400. 02000000:00000001:0.0:1787695830.904858:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695830.904861:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695830.904864:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b318. 00000400:00000010:0.0:1787695830.904867:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96088. 00000100:00000200:0.0:1787695830.904870:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139529984, portal 4 00000100:00000001:0.0:1787695830.904872:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695830.904874:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930149120 : -128555779402496 : ffff8b14466f5500) 00000100:00000040:0.0:1787695830.904879:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5500 x1874531139529984/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695855 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904897:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695830.904899:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695830.904902:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 00000100:00000200:0.0:1787695830.904906:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139529984, offset 0 00000400:00000200:0.0:1787695830.904910:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695830.904914:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695830.904917:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695830.904920:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695830.904923:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695830.904925:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695830.904928:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695830.904931:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16950:16950:256:4294967295] 192.168.201.140@tcp LPNI seq info [16949:16949:8:4294967295] 00000400:00000200:0.0:1787695830.904935:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695830.904941:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695830.904946:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695830.904949:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:0.0:1787695830.904954:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695830.904957:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 2 secs later) 00000800:00000200:0.1:1787695830.904962:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695830.904965:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 2 secs later) 00000800:00000100:0.0:1787695830.904968:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695830.904970:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:0.0:1787695830.904972:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695830.904976:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695830.904979:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:0.0:1787695830.904981:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:0.0:1787695830.904984:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695830.904988:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5500 x1874531139529984/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695855 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.904996:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695830.904999:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5500 x1874531139529984/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695855 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.905018:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.905020:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695830.905024:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695830.905027:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:0.0:1787695830.905030:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695830.905031:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298391069 00000100:00000001:0.0:1787695830.905033:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298402616 00000100:00000001:0.0:1787695830.905035:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298391069 00000100:00000001:0.0:1787695830.905036:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298402616 00000100:00000001:0.0:1787695830.905038:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695830.905039:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.905040:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695830.905043:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695830/real 1787695830] req@ffff8b14466f5500 x1874531139529984/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695830.905048:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695830.905049:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695830.905050:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b318. 00000400:00000200:0.0:1787695830.905052:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96088 00000400:00000010:0.0:1787695830.905053:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96088. 00000100:00000001:0.0:1787695830.905055:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695830.905057:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5500 x1874531139529984/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695830.905062:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5500 x1874531139529984/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.905066:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695830.905067:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.905068:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695830.905069:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695830.905071:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5500 x1874531139529984/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.905080:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695830.905082:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5500 x1874531139529984/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.905087:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695830.905088:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.905089:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695830.905090:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.905091:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695830.905092:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695830.905094:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695830.905096:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695830.905098:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695830.905100:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5500 x1874531139529984/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695830.905105:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139529984:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695830.905107:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695830.905109:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4700 x1874531139530112/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.905113:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695830.905114:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695830.905115:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695830.905116:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695830.905118:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695830.905119:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139530112:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695830.905121:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695830.905123:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695830.905127:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695830.905128:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695830.905130:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5c00. 02000000:00000001:0.0:1787695830.905132:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695830.905134:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695830.905136:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b318. 00000400:00000010:0.0:1787695830.905138:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96088. 00000100:00000200:0.0:1787695830.905140:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139530112, portal 10 00000100:00000001:0.0:1787695830.905141:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695830.905142:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930145536 : -128555779406080 : ffff8b14466f4700) 00000100:00000040:0.0:1787695830.905146:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4700 x1874531139530112/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695855 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.905151:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695830.905152:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695830.905154:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 00000100:00000200:0.0:1787695830.905155:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139530112, offset 0 00000400:00000200:0.0:1787695830.905158:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695830.905160:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695830.905162:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695830.905164:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695830.905166:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695830.905168:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695830.905169:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695830.905171:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16951:16951:256:4294967295] 192.168.201.140@tcp LPNI seq info [16950:16950:8:4294967295] 00000400:00000200:0.0:1787695830.905174:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695830.905178:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695830.905181:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695830.905183:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:0.0:1787695830.905186:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695830.905188:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 2 secs later) 00000800:00000200:0.1:1787695830.905191:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695830.905192:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 2 secs later) 00000800:00000100:0.0:1787695830.905195:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695830.905199:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:0.0:1787695830.905201:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695830.905204:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695830.905206:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:0.0:1787695830.905207:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:0.0:1787695830.905209:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695830.905211:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4700 x1874531139530112/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695855 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.905216:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695830.905218:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4700 x1874531139530112/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695855 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.905223:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.905224:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695830.905226:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695830.905228:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:0.0:1787695830.905230:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695830.905231:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298391255 00000100:00000001:0.0:1787695830.905233:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298402616 00000100:00000001:0.0:1787695830.905234:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298391255 00000100:00000001:0.0:1787695830.905236:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298402616 00000100:00000001:0.0:1787695830.905237:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695830.905238:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.905240:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695830.905242:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695830/real 1787695830] req@ffff8b14466f4700 x1874531139530112/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695830.905246:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695830.905247:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695830.905248:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b318. 00000400:00000200:0.0:1787695830.905249:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96088 00000400:00000010:0.0:1787695830.905250:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96088. 00000100:00000001:0.0:1787695830.905255:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695830.905257:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4700 x1874531139530112/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695830.905263:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4700 x1874531139530112/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.905267:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695830.905267:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.905269:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695830.905269:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695830.905272:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4700 x1874531139530112/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.905276:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695830.905278:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4700 x1874531139530112/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.905282:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695830.905284:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.905285:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695830.905285:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.905287:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695830.905288:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695830.905289:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695830.905291:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695830.905292:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695830.905294:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4700 x1874531139530112/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695830.905299:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139530112:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695830.905301:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695830.905306:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4000 x1874531139530240/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.905311:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695830.905312:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695830.905314:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695830.905315:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695830.905316:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695830.905318:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139530240:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695830.905320:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695830.905321:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695830.905322:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695830.905323:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695830.905325:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5800. 02000000:00000001:0.0:1787695830.905326:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695830.905352:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695830.905355:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b318. 00000400:00000010:0.0:1787695830.905358:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96088. 00000100:00000200:0.0:1787695830.905361:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139530240, portal 10 00000100:00000001:0.0:1787695830.905381:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695830.905384:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930143744 : -128555779407872 : ffff8b14466f4000) 00000100:00000040:0.0:1787695830.905391:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4000 x1874531139530240/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695855 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.905399:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695830.905402:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695830.905405:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 00000100:00000200:0.0:1787695830.905408:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139530240, offset 0 00000400:00000200:0.0:1787695830.905411:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695830.905414:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695830.905417:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695830.905419:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695830.905421:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695830.905422:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695830.905428:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695830.905430:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16952:16952:256:4294967295] 192.168.201.140@tcp LPNI seq info [16951:16951:8:4294967295] 00000400:00000200:0.0:1787695830.905433:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695830.905438:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695830.905441:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695830.905443:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:0.0:1787695830.905445:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695830.905447:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 2 secs later) 00000800:00000200:0.1:1787695830.905450:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695830.905452:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 16, 2 secs later) 00000800:00000100:0.0:1787695830.905454:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695830.905456:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:0.0:1787695830.905459:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695830.905462:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695830.905465:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:0.0:1787695830.905467:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:0.0:1787695830.905469:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695830.905472:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4000 x1874531139530240/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695855 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.905478:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695830.905481:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4000 x1874531139530240/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695855 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.905486:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.905487:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695830.905490:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695830.905492:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:0.0:1787695830.905495:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695830.905497:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298391234 00000100:00000001:0.0:1787695830.905499:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298402616 00000100:00000001:0.0:1787695830.905501:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298391234 00000100:00000001:0.0:1787695830.905502:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298402616 00000100:00000001:0.0:1787695830.905508:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695830.905510:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.905511:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695830.905515:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695830/real 1787695830] req@ffff8b14466f4000 x1874531139530240/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695830.905521:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695830.905522:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695830.905524:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b318. 00000400:00000200:0.0:1787695830.905525:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96088 00000400:00000010:0.0:1787695830.905526:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96088. 00000100:00000001:0.0:1787695830.905528:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695830.905530:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4000 x1874531139530240/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695830.905535:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4000 x1874531139530240/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.905539:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695830.905540:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.905541:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695830.905542:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695830.905544:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4000 x1874531139530240/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.905548:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695830.905550:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4000 x1874531139530240/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.905554:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695830.905556:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.905557:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695830.905558:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.905559:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695830.905560:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695830.905565:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695830.905567:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695830.905569:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695830.905571:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4000 x1874531139530240/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695830.905576:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139530240:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695830.905579:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.905580:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695830.905582:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5500 x1874531139529984/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.905586:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695830.905587:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695830.905588:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5400. 02000000:00000001:0.0:1787695830.905590:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695830.905591:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695830.905592:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:0.0:1787695830.905593:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695830.905594:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675400. 02000000:00000001:0.0:1787695830.905596:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695830.905597:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695830.905599:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5500. 00000100:00000001:0.0:1787695830.905601:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695830.905603:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.905605:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695830.905608:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4700 x1874531139530112/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.905614:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695830.905616:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695830.905617:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5c00. 02000000:00000001:0.0:1787695830.905620:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695830.905621:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695830.905629:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695830.905631:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695830.905633:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:0.0:1787695830.905636:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695830.905637:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695830.905639:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4700. 00000100:00000001:0.0:1787695830.905641:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695830.905643:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.905645:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695830.905649:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4000 x1874531139530240/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695855 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695830.905657:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695830.905658:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695830.905659:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5800. 02000000:00000001:0.0:1787695830.905661:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695830.905662:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695830.905662:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=17 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695830.905664:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695830.905665:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676000. 02000000:00000001:0.0:1787695830.905667:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695830.905667:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695830.905668:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4000. 00000100:00000001:0.0:1787695830.905670:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695830.905670:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.905672:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695830.905674:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695830.905675:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695830.905676:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695830.905678:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695830.966433:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695830.966442:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695830.966443:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695830.966448:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695830.966452:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695830.966455:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695830.966457:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695830.966460:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695830.966463:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695831.414467:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695831.414478:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695831.414480:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695831.414485:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695831.414488:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695831.414494:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695831.414499:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695831.606503:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695831.606509:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695831.606515:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695831.606516:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695831.606520:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695831.606522:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695831.606525:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695831.606527:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695831.606530:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695831.606533:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695831.606536:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695831.606542:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695831.606545:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695831.606546:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695831.606549:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695831.606552:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695831.606554:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695831.606556:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695831.606560:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695831.606561:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695831.606615:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695831.606619:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695831.606621:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695831.990452:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695831.990458:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695831.990460:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695831.990464:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695831.990466:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695831.990470:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695831.990472:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695831.990474:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695831.990476:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695832.438444:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695832.438455:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695832.438457:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695832.438462:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695832.438464:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695832.438468:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695832.438471:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695832.630468:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695832.630474:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695832.630496:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695832.630498:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695832.630500:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695832.630501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695832.630504:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695832.630505:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695832.630507:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695832.630508:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695832.630510:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695832.630511:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695832.630570:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695832.630573:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695832.630576:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695833.014506:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695833.014513:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695833.014515:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695833.014519:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695833.014522:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695833.014525:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695833.014527:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695833.014528:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695833.014530:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695833.462453:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695833.462473:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695833.462476:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695833.462483:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695833.462486:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695833.462494:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695833.462498:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695833.654476:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695833.654484:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695833.654492:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695833.654495:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695833.654499:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695833.654500:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695833.654502:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695833.654504:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695833.654507:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695833.654508:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695833.654509:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695833.654510:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695833.654511:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695833.654512:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695833.654513:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695833.654514:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695833.654517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695833.654518:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695833.654520:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695833.654521:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695833.654523:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695833.654524:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695833.654526:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695833.654527:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695833.654596:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695833.654600:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695833.654602:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695834.038502:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695834.038511:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695834.038513:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695834.038517:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695834.038547:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695834.038552:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695834.038555:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695834.038559:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695834.038562:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695834.486460:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695834.486474:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695834.486477:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695834.486482:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695834.486484:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695834.486490:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695834.486494:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695834.678442:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695834.678448:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695834.678452:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695834.678453:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695834.678471:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695834.678472:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695834.678474:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695834.678475:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695834.678478:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695834.678478:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695834.678482:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695834.678482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695834.678544:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695834.678549:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695834.678561:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695835.062464:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695835.062471:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695835.062472:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695835.062477:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695835.062480:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695835.062483:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695835.062484:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695835.062486:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695835.062489:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695835.510430:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695835.510442:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695835.510444:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695835.510445:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695835.510450:0:2405:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787695835.510451:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695835.510454:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695835.510459:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000400:3.0:1787695835.510465:0:2405:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695750/real 1787695750] req@ffff8b14465adc00 x1874531139517824/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695835 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695835.510467:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787695835.510481:0:2405:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787695835.510484:0:2405:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1787695835.510487:0:2405:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14456d12c0. 00000400:00000200:3.0:1787695835.510493:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b145078ad48 00000400:00000010:3.0:1787695835.510495:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b145078ad48. 00000100:00000001:3.0:1787695835.510498:0:2405:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787695835.510501:0:2405:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465adc00 x1874531139517824/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695835 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1787695835.510507:0:2405:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465adc00 x1874531139517824/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695835 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695835.510512:0:2405:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787695835.510514:0:2405:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695835.510516:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695835.510517:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787695835.510520:0:2405:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b14465adc00 x1874531139517824/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695835 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695835.510546:0:2405:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:3.0:1787695835.510548:0:2405:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b144515c000 already not connected (conn 21, was 11): DISCONN 00000100:00000001:3.0:1787695835.510550:0:2405:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:3.0:1787695835.510551:0:2405:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695835.510553:0:2405:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000400:3.0:1787695835.510556:0:2405:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695750/real 1787695750] req@ffff8b14465ad180 x1874531139517696/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695835 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787695835.510562:0:2405:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787695835.510562:0:2405:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1787695835.510564:0:2405:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14456d1a50. 00000400:00000200:3.0:1787695835.510565:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b145078a880 00000400:00000010:3.0:1787695835.510567:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b145078a880. 00000100:00000001:3.0:1787695835.510569:0:2405:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787695835.510570:0:2405:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ad180 x1874531139517696/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695835 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1787695835.510575:0:2405:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ad180 x1874531139517696/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695835 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695835.510579:0:2405:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787695835.510580:0:2405:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695835.510581:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695835.510582:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787695835.510584:0:2405:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b14465ad180 x1874531139517696/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695835 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695835.510588:0:2405:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:3.0:1787695835.510589:0:2405:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b1458081800 already not connected (conn 25, was 15): DISCONN 00000100:00000001:3.0:1787695835.510590:0:2405:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:3.0:1787695835.510591:0:2405:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695835.510594:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695835.510595:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695835.510597:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695835.510637:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695835.510640:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695835.510642:0:2405:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695835.510643:0:2405:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695835.510647:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465adc00 x1874531139517824/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695835 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695835.510653:0:2405:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:3.0:1787695835.510655:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695835.510656:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695835.510657:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695835.510660:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465adc00 x1874531139517824/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695835 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695835.510665:0:2405:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:608c526c-eb6e-4a54-80a5-8969828241eb:2405:1874531139517824:192.168.201.140@tcp:400:kworker.0 00000100:00000001:3.0:1787695835.510668:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695835.510669:0:2405:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695835.510670:0:2405:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695835.510672:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ad180 x1874531139517696/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695835 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695835.510677:0:2405:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:3.0:1787695835.510678:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695835.510680:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695835.510681:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695835.510683:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ad180 x1874531139517696/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695835 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695835.510687:0:2405:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2405:1874531139517696:192.168.201.140@tcp:400:kworker.0 00000100:00000001:3.0:1787695835.510690:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695835.510692:0:2405:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695835.510694:0:2405:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465adc00 x1874531139517824/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695835 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695835.510702:0:2405:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787695835.510704:0:2405:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1787695835.510708:0:2405:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1446859000. 02000000:00000001:3.0:1787695835.510712:0:2405:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787695835.510715:0:2405:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1787695835.510716:0:2405:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=16 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:3.0:1787695835.510718:0:2405:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1787695835.510720:0:2405:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1442c54300. 02000000:00000001:3.0:1787695835.510724:0:2405:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787695835.510725:0:2405:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1787695835.510726:0:2405:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465adc00. 00000100:00000001:3.0:1787695835.510729:0:2405:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787695835.510730:0:2405:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695835.510731:0:2405:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695835.510733:0:2405:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ad180 x1874531139517696/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695835 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695835.510737:0:2405:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787695835.510738:0:2405:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1787695835.510739:0:2405:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1446858600. 02000000:00000001:3.0:1787695835.510741:0:2405:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787695835.510742:0:2405:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1787695835.510742:0:2405:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=15 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:3.0:1787695835.510743:0:2405:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1787695835.510744:0:2405:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1442c54100. 02000000:00000001:3.0:1787695835.510745:0:2405:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787695835.510746:0:2405:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1787695835.510747:0:2405:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ad180. 00000100:00000001:3.0:1787695835.510748:0:2405:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787695835.510749:0:2405:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695835.510751:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695835.702462:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695835.702468:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695835.702473:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695835.702474:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695835.702477:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695835.702477:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695835.702481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695835.702482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695835.702484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695835.702485:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695835.702487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695835.702488:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695835.702537:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695835.702540:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695835.702543:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787695836.022596:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695836.022620:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787695836.022623:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695836.022626:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695836.022631:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695836.022638:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3723 00000400:00000200:3.0:1787695836.022648:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695836.022655:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695836.022658:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695836.022661:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695836.022664:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695836.022667:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695836.022670:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695836.022674:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695836.022677:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:3.0:1787695836.022680:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695836.022683:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787695836.022686:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787695836.022689:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787695836.022692:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695836.022694:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695836.022697:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695836.022699:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695836.022702:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787695836.022705:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695836.022708:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695836.022711:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695836.022715:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695836.022718:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695836.022720:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787695836.022722:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695836.022729:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4a80. 00000020:00000040:3.0:1787695836.022738:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787695836.022743:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695836.022745:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695836.022748:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695836.022751:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695836.022755:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695836.022762:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675800. 00000100:00000001:3.0:1787695836.022769:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695836.022780:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4a80 x1874531139530368/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695836.022792:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695836.022797:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695836.022802:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ad0d67] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4a80 x1874531139530368/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695836.022824:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695836.022828:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695836.022831:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695836.022833:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695836.022834:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695836.022837:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695836.022840:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3723 00000400:00000200:3.0:1787695836.022844:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695836.022848:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695836.022851:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695836.022855:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695836.022858:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695836.022860:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695836.022862:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695836.022865:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787695836.022866:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:3.0:1787695836.022867:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:3.0:1787695836.022872:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787695836.022874:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695836.022878:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1787695836.022882:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:3.0:1787695836.022888:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:0.0:1787695836.022890:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4a80 x1874531139530368/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695836.022892:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:3.0:1787695836.022901:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695836.022905:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695836.022906:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695836.022908:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695836.022912:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695836.022913:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695836.022918:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695836.022918:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:0.0:1787695836.022922:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695836.022924:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00100000:0.0:1787695836.022928:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139530368:192.168.201.140@tcp:250:kworker.0 00000100:00080000:3.0:1787695836.022932:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695836.022934:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787695836.022938:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695836.022940:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787695836.022942:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787695836.022943:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695836.022947:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787695836.022949:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:0.0:1787695836.022952:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5000. 02000000:00000001:3.0:1787695836.022954:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695836.022955:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695836.022959:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:0.0:1787695836.022963:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000008:00100000:3.0:1787695836.022965:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000010:0.0:1787695836.022968:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b318. 00000008:00000001:3.0:1787695836.022970:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695836.022973:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695836.022975:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96088. 00000100:00000010:3.0:1787695836.022978:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7100. 00000100:00000200:0.0:1787695836.022982:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139530368, portal 25 00000020:00000040:3.0:1787695836.022983:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000100:00000001:0.0:1787695836.022986:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695836.022989:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930146432 : -128555779405184 : ffff8b14466f4a80) 02000000:00000001:3.0:1787695836.022989:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695836.022993:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695836.022998:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4a80 x1874531139530368/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695836.022999:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695836.023002:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695836.023006:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.023008:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695836.023012:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695836.023016:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96e58. 02000000:00000010:3.0:1787695836.023016:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676000. 00000100:00000001:3.0:1787695836.023022:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695836.023030:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7100 x1874531139530496/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1787695836.023035:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139530368, offset 0 00000100:00000001:3.0:1787695836.023044:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695836.023048:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695836.023051:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695836.023056:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000004e8abd7] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7100 x1874531139530496/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695836.023066:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695836.023077:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:3.0:1787695836.023087:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695836.023091:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695836.023095:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695836.023096:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695836.023099:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1787695836.023101:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1787695836.023104:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:0.0:1787695836.023111:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00080000:3.0:1787695836.023111:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3723 00000400:00000200:0.0:1787695836.023120:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1787695836.023121:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695836.023128:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:3.0:1787695836.023130:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695836.023134:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1787695836.023137:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16953:16953:256:4294967295] 192.168.201.140@tcp LPNI seq info [16952:16952:8:4294967295] 00000100:00000040:3.0:1787695836.023141:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695836.023147:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1787695836.023148:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695836.023152:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695836.023157:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695836.023160:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695836.023163:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000400:00000200:0.0:1787695836.023168:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000020:00000001:3.0:1787695836.023168:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695836.023173:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000800:00000200:0.0:1787695836.023180:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000020:00000001:3.0:1787695836.023180:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:3.0:1787695836.023186:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000800:00000010:0.0:1787695836.023188:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000100:00000040:3.0:1787695836.023193:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000800:00000200:0.0:1787695836.023200:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787695836.023200:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695836.023206:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000200:0.1:1787695836.023208:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787695836.023213:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695836.023218:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:3.0:1787695836.023223:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695836.023225:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695836.023226:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695836.023231:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695836.023233:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298407736 00000100:00000001:0.0:1787695836.023239:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298402616 00000100:00000001:0.0:1787695836.023242:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298407736 02000000:00000001:3.0:1787695836.023246:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787695836.023247:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298402616 00000100:00000001:0.0:1787695836.023265:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695836.023266:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695836.023269:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695836.023272:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.023274:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695836.023276:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787695836.023278:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1787695836.023280:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695836.023285:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.023286:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695836.023287:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.023289:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787695836.023294:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:3.0:1787695836.023294:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7800. 00000100:00000001:0.0:1787695836.023296:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695836.023300:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1787695836.023301:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=16 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000100:00000040:0.0:1787695836.023315:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7100 x1874531139530496/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695836.023317:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695836.023320:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695836.023323:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695836.023326:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787695836.023333:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695836.023335:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695836.023338:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695836.023340:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695836.023350:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695836.023354:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139530496:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695836.023357:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695836.023359:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695836.023360:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695836.023362:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695836.023387:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f4400. 02000000:00000001:0.0:1787695836.023390:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695836.023395:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695836.023399:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b580. 00000400:00000010:0.0:1787695836.023404:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695836.023408:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139530496, portal 10 00000100:00000001:0.0:1787695836.023410:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695836.023412:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930156288 : -128555779395328 : ffff8b14466f7100) 02000000:00000001:3.0:1787695836.023419:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695836.023422:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7100 x1874531139530496/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1787695836.023427:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 00000100:00000001:0.0:1787695836.023435:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787695836.023435:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695836.023442:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00100000:3.0:1787695836.023446:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7800 x1874531139530624/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:0.0:1787695836.023448:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f962a8. 00000100:00000200:0.0:1787695836.023453:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139530496, offset 0 00000400:00000200:0.0:1787695836.023463:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695836.023466:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1787695836.023469:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695836.023472:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695836.023476:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695836.023481:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1787695836.023482:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7c57981] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7800 x1874531139530624/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695836.023485:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695836.023490:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695836.023495:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695836.023502:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16954:16954:255:4294967295] 192.168.201.140@tcp LPNI seq info [16953:16953:7:4294967295] 00000100:00000001:3.0:1787695836.023508:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695836.023509:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:3.0:1787695836.023515:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1787695836.023522:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00080000:3.0:1787695836.023524:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000200:0.0:1787695836.023529:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695836.023529:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695836.023533:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695836.023535:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695836.023539:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3723 00000400:00000200:3.0:1787695836.023544:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695836.023548:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695836.023551:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695836.023561:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695836.023563:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695836.023565:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695836.023567:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695836.023570:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695836.023571:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:3.0:1787695836.023573:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695836.023575:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:3.0:1787695836.023578:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000800:00000010:0.0:1787695836.023582:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981300. 00000020:00000001:3.0:1787695836.023583:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000800:00000200:0.0:1787695836.023591:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:3.0:1787695836.023592:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695836.023598:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.1:1787695836.023601:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:3.0:1787695836.023605:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695836.023606:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695836.023611:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000040:0.0:1787695836.023612:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000020:00000040:3.0:1787695836.023617:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000400:00000200:0.0:1787695836.023621:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:3.0:1787695836.023625:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695836.023626:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298397638 00000100:00000001:0.0:1787695836.023631:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298407736 00000100:00000001:3.0:1787695836.023633:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.023634:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298397638 02000000:00000001:3.0:1787695836.023638:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787695836.023639:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298407736 00000100:00000001:0.0:1787695836.023643:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695836.023645:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695836.023645:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695836.023650:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695836.023651:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.023652:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695836.023657:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787695836.023658:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000008:00100000:3.0:1787695836.023661:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787695836.023662:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787695836.023665:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:3.0:1787695836.023668:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.023669:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695836.023673:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695836.023677:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5c00. 00000100:00000001:0.0:1787695836.023680:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1787695836.023681:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000100:00000040:0.0:1787695836.023687:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7800 x1874531139530624/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695836.023688:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695836.023691:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695836.023697:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695836.023701:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787695836.023702:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787695836.023705:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.023706:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695836.023712:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000010:3.0:1787695836.023713:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675400. 02000000:00000001:0.0:1787695836.023715:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695836.023718:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695836.023722:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139530624:192.168.201.140@tcp:38:kworker.0 00000100:00000001:3.0:1787695836.023726:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.023729:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695836.023733:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:3.0:1787695836.023735:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5c00 x1874531139530752/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787695836.023736:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695836.023742:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695836.023748:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5400. 00000100:00000001:3.0:1787695836.023750:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1787695836.023753:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695836.023755:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695836.023759:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005bfbcee1] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5c00 x1874531139530752/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695836.023768:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695836.023773:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bf78. 00000400:00000010:0.0:1787695836.023778:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96990. 00000100:00000001:3.0:1787695836.023781:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695836.023783:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139530624, portal 10 00000100:00080000:3.0:1787695836.023786:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695836.023787:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695836.023789:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930158080 : -128555779393536 : ffff8b14466f7800) 00000100:00080000:3.0:1787695836.023796:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000040:0.0:1787695836.023800:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7800 x1874531139530624/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695836.023800:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695836.023804:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695836.023812:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787695836.023813:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787695836.023818:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3723 00000100:00000040:0.0:1787695836.023819:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695836.023823:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f963b8. 00000100:00000200:0.0:1787695836.023826:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139530624, offset 0 00000400:00000200:3.0:1787695836.023829:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695836.023834:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:3.0:1787695836.023838:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000200:0.0:1787695836.023839:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:3.0:1787695836.023844:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1787695836.023846:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695836.023851:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1787695836.023852:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695836.023855:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:3.0:1787695836.023859:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1787695836.023860:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695836.023867:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000040:3.0:1787695836.023867:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695836.023871:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16955:16955:254:4294967295] 192.168.201.140@tcp LPNI seq info [16954:16954:6:4294967295] 00000100:00000001:3.0:1787695836.023874:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:0.0:1787695836.023878:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:3.0:1787695836.023881:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695836.023886:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000400:00000200:0.0:1787695836.023888:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000020:00000001:3.0:1787695836.023889:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000200:0.0:1787695836.023893:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000020:00000040:3.0:1787695836.023895:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000800:00000010:0.0:1787695836.023897:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b144a720700. 00000020:00000001:3.0:1787695836.023900:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000800:00000200:0.0:1787695836.023903:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000020:00000001:3.0:1787695836.023906:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000800:00000200:0.1:1787695836.023910:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:3.0:1787695836.023915:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695836.023916:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695836.023922:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695836.023923:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:0.0:1787695836.023926:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695836.023927:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298396486 00000100:00000001:0.0:1787695836.023929:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298407736 00000100:00000001:0.0:1787695836.023933:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298396486 00000100:00000040:3.0:1787695836.023934:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695836.023938:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695836.023944:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298407736 00000020:00000040:3.0:1787695836.023944:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00000001:0.0:1787695836.023948:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1787695836.023951:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695836.023952:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.023957:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695836.023958:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.023959:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787695836.023964:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787695836.023965:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695836.023968:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787695836.023970:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787695836.023970:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695836.023973:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695836.023976:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787695836.023977:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695836.023982:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000040:0.0:1787695836.023983:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5c00 x1874531139530752/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787695836.023987:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787695836.023994:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.023996:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695836.023999:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695836.023999:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695836.024005:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695836.024007:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1787695836.024008:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6680. 02000000:00000001:0.0:1787695836.024009:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695836.024016:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139530752:192.168.201.140@tcp:8:kworker.0 00000020:00000040:3.0:1787695836.024016:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:3.0:1787695836.024022:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787695836.024024:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787695836.024025:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695836.024028:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787695836.024030:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695836.024032:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695836.024033:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787695836.024037:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787695836.024039:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695836.024044:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5c00. 02000000:00000010:3.0:1787695836.024047:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 02000000:00000001:0.0:1787695836.024049:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695836.024053:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695836.024058:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695836.024065:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b738. 00000100:00100000:3.0:1787695836.024065:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6680 x1874531139530880/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:0.0:1787695836.024073:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96440. 00000100:00000200:0.0:1787695836.024078:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139530752, portal 4 00000100:00000001:3.0:1787695836.024080:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695836.024082:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695836.024086:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150912 : -128555779400704 : ffff8b14466f5c00) 00000100:00000001:3.0:1787695836.024086:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695836.024096:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f6d2abf9] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6680 x1874531139530880/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695836.024100:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5c00 x1874531139530752/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695836.024121:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.024123:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787695836.024126:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695836.024133:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000040:0.0:1787695836.024136:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695836.024137:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:0.0:1787695836.024140:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96d48. 00000100:00000200:0.0:1787695836.024143:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139530752, offset 0 00000100:00080000:3.0:1787695836.024147:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1787695836.024151:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695836.024155:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:0.0:1787695836.024158:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:3.0:1787695836.024162:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3723 00000400:00000200:0.0:1787695836.024164:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695836.024170:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695836.024174:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:3.0:1787695836.024174:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695836.024179:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695836.024184:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:3.0:1787695836.024185:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000200:0.0:1787695836.024189:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16956:16956:253:4294967295] 192.168.201.140@tcp LPNI seq info [16955:16955:5:4294967295] 00000100:00080000:3.0:1787695836.024191:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1787695836.024196:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695836.024200:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695836.024206:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1787695836.024207:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000040:3.0:1787695836.024213:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000200:0.0:1787695836.024217:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695836.024219:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000010:0.0:1787695836.024223:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfc900. 00000020:00000001:3.0:1787695836.024225:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695836.024228:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000800:00000200:0.0:1787695836.024229:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000020:00000001:3.0:1787695836.024233:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000200:0.1:1787695836.024236:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695836.024239:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695836.024239:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000100:00000040:0.0:1787695836.024244:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000020:00000001:3.0:1787695836.024246:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000200:0.0:1787695836.024265:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695836.024269:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298396516 00000020:00000001:3.0:1787695836.024269:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:0.0:1787695836.024274:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298407736 00000100:00000040:3.0:1787695836.024278:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695836.024279:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298396516 00000100:00000001:0.0:1787695836.024283:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298407736 00000100:00000001:3.0:1787695836.024285:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695836.024287:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1787695836.024290:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695836.024292:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695836.024294:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695836.024298:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695836.024299:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00000001:0.0:1787695836.024305:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695836.024306:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695836.024311:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.024312:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1787695836.024314:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787695836.024316:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:3.0:1787695836.024320:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695836.024321:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695836.024324:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787695836.024326:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.024333:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695836.024338:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6680 x1874531139530880/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695836.024343:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695836.024344:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695836.024346:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695836.024348:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695836.024349:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695836.024351:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139530880:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695836.024353:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695836.024354:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695836.024356:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695836.024357:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695836.024359:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f5800. 02000000:00000001:0.0:1787695836.024360:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695836.024379:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695836.024383:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617bb00. 00000400:00000010:0.0:1787695836.024388:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96f68. 00000100:00000200:0.0:1787695836.024391:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139530880, portal 4 00000100:00000001:0.0:1787695836.024393:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695836.024394:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930153600 : -128555779398016 : ffff8b14466f6680) 00000100:00000040:0.0:1787695836.024401:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6680 x1874531139530880/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695836.024408:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695836.024412:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787695836.024416:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:3.0:1787695836.024417:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695836.024422:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695836.024423:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695836.024427:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96dd0. 00000100:00000200:0.0:1787695836.024430:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139530880, offset 0 00000100:00000010:3.0:1787695836.024434:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5880. 00000020:00000040:3.0:1787695836.024438:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000400:00000200:0.0:1787695836.024439:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695836.024445:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1787695836.024446:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787695836.024449:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695836.024452:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:3.0:1787695836.024455:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:0.0:1787695836.024457:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1787695836.024459:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:0.0:1787695836.024461:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 02000000:00000001:3.0:1787695836.024464:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695836.024465:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 02000000:00000010:3.0:1787695836.024471:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 00000400:00000200:0.0:1787695836.024475:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:3.0:1787695836.024478:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695836.024480:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16957:16957:252:4294967295] 192.168.201.140@tcp LPNI seq info [16956:16956:4:4294967295] 00000400:00000200:0.0:1787695836.024488:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00100000:3.0:1787695836.024490:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5880 x1874531139531008/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1787695836.024500:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695836.024512:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695836.024516:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000010:0.0:1787695836.024518:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfca00. 00000100:00000001:3.0:1787695836.024522:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1787695836.024524:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695836.024532:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:3.0:1787695836.024533:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000827e7dbe] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5880 x1874531139531008/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695836.024534:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695836.024539:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:0.0:1787695836.024546:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695836.024549:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298396189 00000100:00000001:0.0:1787695836.024554:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298407736 00000100:00000001:0.0:1787695836.024557:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298396189 00000100:00000001:3.0:1787695836.024559:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.024560:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298407736 00000100:00000001:0.0:1787695836.024565:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1787695836.024566:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695836.024567:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.024571:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695836.024573:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787695836.024575:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695836.024578:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787695836.024580:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1787695836.024582:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695836.024583:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787695836.024587:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695836.024590:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695836.024590:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787695836.024593:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1787695836.024598:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3723 00000100:00000040:0.0:1787695836.024599:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5880 x1874531139531008/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695836.024611:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695836.024612:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695836.024614:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695836.024618:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695836.024621:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1787695836.024621:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 02000000:00000001:0.0:1787695836.024626:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695836.024628:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787695836.024631:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139531008:192.168.201.140@tcp:38:kworker.0 00000100:00000040:3.0:1787695836.024635:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695836.024640:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787695836.024641:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787695836.024643:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695836.024646:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695836.024647:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695836.024648:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695836.024653:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b14464f7400. 00000100:00000001:3.0:1787695836.024653:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:0.0:1787695836.024656:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695836.024659:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:0.0:1787695836.024661:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695836.024663:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617b058. 00000400:00000010:0.0:1787695836.024666:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96198. 00000020:00000040:3.0:1787695836.024668:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000100:00000200:0.0:1787695836.024670:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139531008, portal 10 00000020:00000001:3.0:1787695836.024673:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787695836.024674:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695836.024677:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150016 : -128555779401600 : ffff8b14466f5880) 00000020:00000040:3.0:1787695836.024678:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000100:00000040:0.0:1787695836.024684:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5880 x1874531139531008/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695836.024684:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000020:00000001:3.0:1787695836.024690:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:0.0:1787695836.024698:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1787695836.024701:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:0.0:1787695836.024703:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695836.024706:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:0.0:1787695836.024708:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96770. 00000100:00000040:3.0:1787695836.024712:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000200:0.0:1787695836.024714:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139531008, offset 0 00000100:00000001:3.0:1787695836.024716:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695836.024721:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000400:00000200:0.0:1787695836.024726:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00080000:3.0:1787695836.024729:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:0.0:1787695836.024735:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695836.024735:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695836.024740:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:0.0:1787695836.024744:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 02000000:00000001:3.0:1787695836.024746:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1787695836.024750:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1787695836.024751:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695836.024756:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:3.0:1787695836.024757:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:0.0:1787695836.024762:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000008:00100000:3.0:1787695836.024762:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1787695836.024768:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000008:00000001:3.0:1787695836.024769:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695836.024772:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16958:16958:251:4294967295] 192.168.201.140@tcp LPNI seq info [16957:16957:3:4294967295] 00000100:00000001:3.0:1787695836.024774:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695836.024779:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000010:3.0:1787695836.024783:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4e00. 00000400:00000200:0.0:1787695836.024788:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000020:00000040:3.0:1787695836.024789:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=17 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:3.0:1787695836.024796:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000200:0.0:1787695836.024797:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695836.024800:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1787695836.024803:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfc700. 00000100:00000001:3.0:1787695836.024805:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:0.0:1787695836.024810:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:3.0:1787695836.024810:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695836.024814:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1787695836.024821:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000010:3.0:1787695836.024822:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 00000100:00000001:0.0:1787695836.024824:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695836.024829:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000100:00000001:3.0:1787695836.024829:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695836.024836:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695836.024837:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298396375 00000100:00100000:3.0:1787695836.024840:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4e00 x1874531139531136/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695836.024841:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298407736 00000100:00000001:0.0:1787695836.024845:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298396375 00000100:00000001:0.0:1787695836.024850:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298407736 00000100:00000001:0.0:1787695836.024855:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695836.024859:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695836.024862:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695836.024864:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.024867:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695836.024868:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.024870:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695836.024872:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787695836.024874:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695836.024875:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695836.024876:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.024881:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695836.024885:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcd77cd6] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4e00 x1874531139531136/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695836.024898:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695836.024901:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787695836.024905:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695836.024909:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695836.024911:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1787695836.024913:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1787695836.024914:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695836.024917:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1787695836.024921:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3733) 00000100:00000040:0.0:1787695836.024923:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4e00 x1874531139531136/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695836.024930:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695836.024932:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695836.024933:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695836.024934:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695836.024936:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695836.024938:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139531136:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695836.024941:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695836.024942:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695836.024944:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695836.024945:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695836.024948:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b145101dc00. 02000000:00000001:0.0:1787695836.024954:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695836.024956:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695836.024958:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617be18. 00000400:00000010:0.0:1787695836.024961:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00000200:0.0:1787695836.024963:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139531136, portal 10 00000100:00000001:0.0:1787695836.024965:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695836.024965:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930147328 : -128555779404288 : ffff8b14466f4e00) 00000100:00000040:0.0:1787695836.024970:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4e00 x1874531139531136/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695836.024975:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695836.024976:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695836.024978:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96550. 00000100:00000200:0.0:1787695836.024979:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139531136, offset 0 00000400:00000200:0.0:1787695836.024982:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695836.024985:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695836.024987:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695836.024989:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695836.024991:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695836.024992:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695836.024994:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695836.024996:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16959:16959:250:4294967295] 192.168.201.140@tcp LPNI seq info [16958:16958:2:4294967295] 00000400:00000200:0.0:1787695836.024999:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695836.025003:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695836.025006:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695836.025008:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfc200. 00000800:00000200:0.0:1787695836.025011:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695836.025013:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695836.025015:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695836.025017:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:0.0:1787695836.025020:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695836.025022:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298396353 00000100:00000001:0.0:1787695836.025024:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298407736 00000100:00000001:0.0:1787695836.025031:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298396353 00000100:00000001:0.0:1787695836.025034:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298407736 00000100:00000001:0.0:1787695836.025036:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695836.025039:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.025042:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.025044:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695836.025047:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695836.025049:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787695836.025051:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695836.025052:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695836.025054:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695836.025056:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695836.086431:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695836.086439:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695836.086441:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695836.086445:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695836.086447:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695836.086450:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695836.086453:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695836.086455:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695836.086457:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695836.534427:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695836.534442:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695836.534446:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695836.534452:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695836.534454:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695836.534455:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695836.534463:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695836.534465:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695836.534467:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695836.534469:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695836.534475:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695836.534477:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695836.534481:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695836.534495:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695836.726468:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695836.726474:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695836.726478:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695836.726479:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695836.726482:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695836.726485:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695836.726487:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695836.726492:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695836.726494:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695836.726495:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695836.726498:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695836.726500:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695836.726501:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695836.726503:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695836.726506:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695836.726507:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695836.726507:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695836.726508:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695836.726509:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695836.726524:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695836.726525:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695836.726526:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695836.726527:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695836.726529:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695836.726532:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695836.726532:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695836.726536:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695836.726537:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695836.726538:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695836.726540:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695836.726541:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695836.726542:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695836.726544:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695836.726545:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695836.726547:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695836.726549:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695836.726611:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695836.726623:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695836.726628:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695837.110479:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695837.110489:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695837.110491:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695837.110497:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695837.110500:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695837.110504:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695837.110507:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695837.110510:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695837.110512:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695837.558439:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695837.558439:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695837.558451:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695837.558452:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695837.558456:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695837.558456:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695837.558461:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695837.558463:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695837.558464:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695837.558466:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695837.558469:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695837.558472:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695837.558473:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695837.558479:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695837.750485:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695837.750492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695837.750498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695837.750499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695837.750504:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695837.750505:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695837.750509:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695837.750510:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695837.750513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695837.750515:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695837.750518:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695837.750520:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695837.750592:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695837.750596:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695837.750600:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787695837.878710:0:2391:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.140:988 00000400:00000100:1.0:1787695837.878750:0:2391:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.140@tcp at host 192.168.201.140 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:1.0:1787695837.878759:0:2391:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695837.878763:0:2391:0:(router.c:1791:lnet_notify()) 192.168.201.40@tcp notifying 192.168.201.140@tcp: down 00000400:00000200:1.0:1787695837.878768:0:2391:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695837.878772:0:2391:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:1.0:1787695837.878777:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:1.0:1787695837.878782:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695837.878804:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1787695837.878809:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1787695837.878816:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96e58 00000400:00000010:1.0:1787695837.878819:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96e58. 00000100:00000001:1.0:1787695837.878823:0:2391:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695837.878833:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f4a80 x1874531139530368/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695837.878861:0:2391:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695837.878867:0:2391:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4a80 x1874531139530368/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695861 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695837.878879:0:2391:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695837.878881:0:2391:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787695837.878892:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:1.0:1787695837.878896:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981300. 00000400:00000200:1.0:1787695837.878899:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1787695837.878904:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1787695837.878908:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f962a8 00000400:00000010:1.0:1787695837.878910:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f962a8. 00000100:00000001:1.0:1787695837.878914:0:2391:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787695837.878918:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:1.0:1787695837.878920:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f7100 x1874531139530496/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.878924:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695837.878931:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1787695837.878934:0:2391:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695837.878938:0:2391:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7100 x1874531139530496/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695861 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695837.878944:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695836/real 1787695837] req@ffff8b14466f4a80 x1874531139530368/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695837.878962:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1787695837.878963:0:2391:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695837.878967:0:2391:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000001:0.0:1787695837.878968:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695837.878974:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b318. 00000400:00000200:0.0:1787695837.878980:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96088 00000800:00000100:1.0:1787695837.878982:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000400:00000010:0.0:1787695837.878984:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96088. 00000800:00000010:1.0:1787695837.878987:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b144a720700. 00000100:00000001:0.0:1787695837.878990:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:1.0:1787695837.878995:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000200:0.0:1787695837.878999:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4a80 x1874531139530368/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695837.879004:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1787695837.879011:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f963b8 00000100:00000200:0.0:1787695837.879018:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4a80 x1874531139530368/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1787695837.879023:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f963b8. 00000100:00000001:1.0:1787695837.879031:0:2391:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695837.879038:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f7800 x1874531139530624/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695837.879071:0:2391:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787695837.879073:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695837.879078:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695837.879078:0:2391:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7800 x1874531139530624/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695861 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.879085:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695837.879089:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695837.879091:0:2391:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695837.879095:0:2391:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:0.0:1787695837.879099:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4a80 x1874531139530368/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1787695837.879106:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:1.0:1787695837.879111:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfc900. 00000400:00000200:1.0:1787695837.879117:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:1.0:1787695837.879127:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000100:00000001:0.0:1787695837.879128:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1787695837.879131:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96d48 00000400:00000010:1.0:1787695837.879134:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96d48. 00000100:00000040:0.0:1787695837.879138:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4a80 x1874531139530368/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695837.879140:0:2391:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695837.879148:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f5c00 x1874531139530752/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.879160:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695837.879168:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695837.879170:0:2391:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787695837.879171:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695837.879175:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695837.879176:0:2391:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5c00 x1874531139530752/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695861 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.879179:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695837.879184:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695837.879188:0:2391:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695837.879191:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1787695837.879192:0:2391:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787695837.879201:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:1.0:1787695837.879204:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfca00. 00000400:00000200:1.0:1787695837.879208:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00080000:0.0:1787695837.879214:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000400:00000200:1.0:1787695837.879215:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:1.0:1787695837.879234:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96dd0 00000400:00000010:1.0:1787695837.879236:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96dd0. 00000100:00000001:0.0:1787695837.879241:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1787695837.879242:0:2391:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695837.879250:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f6680 x1874531139530880/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695837.879253:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4a80 x1874531139530368/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695837.879274:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139530368:192.168.201.140@tcp:250:kworker.0 00000100:00000001:0.0:1787695837.879283:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695837.879288:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695837.879294:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4a80 x1874531139530368/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695837.879296:0:2391:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695837.879305:0:2391:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6680 x1874531139530880/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695861 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.879306:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695837.879311:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1787695837.879316:0:2391:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695837.879317:0:2391:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000010:0.0:1787695837.879325:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5000. 00000800:00000100:1.0:1787695837.879326:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:1.0:1787695837.879332:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfc700. 02000000:00000001:0.0:1787695837.879336:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:1.0:1787695837.879337:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000020:00000001:0.0:1787695837.879341:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695837.879346:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000400:00000200:1.0:1787695837.879346:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000020:00000001:0.0:1787695837.879353:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000400:00000200:1.0:1787695837.879355:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96770 02000000:00000010:0.0:1787695837.879357:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675800. 00000400:00000010:1.0:1787695837.879360:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96770. 02000000:00000001:0.0:1787695837.879423:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1787695837.879424:0:2391:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1787695837.879429:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000200:1.0:1787695837.879434:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f5880 x1874531139531008/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1787695837.879435:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4a80. 00000100:00000001:0.0:1787695837.879442:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695837.879446:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695837.879450:0:2391:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787695837.879451:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695837.879455:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1787695837.879457:0:2391:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5880 x1874531139531008/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695861 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.879459:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695837.879462:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695837.879472:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695836/real 1787695837] req@ffff8b14466f5880 x1874531139531008/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695837.879473:0:2391:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695837.879478:0:2391:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787695837.879487:0:2391:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000400:00000001:0.0:1787695837.879490:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000010:1.0:1787695837.879491:0:2391:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfc200. 00000400:00000001:0.0:1787695837.879494:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1787695837.879497:0:2391:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000010:0.0:1787695837.879499:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b058. 00000400:00000200:1.0:1787695837.879504:0:2391:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1787695837.879505:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96198 00000400:00000200:1.0:1787695837.879509:0:2391:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96550 00000400:00000010:0.0:1787695837.879510:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96198. 00000400:00000010:1.0:1787695837.879514:0:2391:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96550. 00000100:00000001:0.0:1787695837.879516:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695837.879522:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5880 x1874531139531008/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695837.879527:0:2391:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695837.879532:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5880 x1874531139531008/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695837.879537:0:2391:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f4e00 x1874531139531136/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695861 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.879549:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695837.879553:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695837.879554:0:2391:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787695837.879556:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695837.879559:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695837.879561:0:2391:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4e00 x1874531139531136/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695861 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695837.879567:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5880 x1874531139531008/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.879588:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695837.879592:0:2391:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695837.879595:0:2391:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1787695837.879606:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5880 x1874531139531008/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.879615:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695837.879618:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.879620:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695837.879621:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.879625:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695837.879630:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695837.879633:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695837.879637:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695837.879639:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695837.879644:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5880 x1874531139531008/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695837.879652:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139531008:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695837.879657:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695837.879662:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695836/real 1787695837] req@ffff8b14466f6680 x1874531139530880/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695837.879671:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695837.879672:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695837.879674:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bb00. 00000400:00000200:0.0:1787695837.879678:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96f68 00000400:00000010:0.0:1787695837.879680:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96f68. 00000100:00000001:0.0:1787695837.879683:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695837.879686:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6680 x1874531139530880/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695837.879693:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6680 x1874531139530880/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.879701:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695837.879709:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.879710:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695837.879711:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695837.879715:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6680 x1874531139530880/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.879722:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695837.879726:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6680 x1874531139530880/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.879733:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695837.879735:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.879737:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695837.879739:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.879741:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695837.879743:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695837.879745:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695837.879749:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695837.879752:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695837.879756:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6680 x1874531139530880/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695837.879763:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139530880:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695837.879768:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695837.879772:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695836/real 1787695837] req@ffff8b14466f5c00 x1874531139530752/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695837.879781:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695837.879782:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695837.879784:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b738. 00000400:00000200:0.0:1787695837.879787:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96440 00000400:00000010:0.0:1787695837.879789:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96440. 00000100:00000001:0.0:1787695837.879797:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695837.879801:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5c00 x1874531139530752/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695837.879811:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5c00 x1874531139530752/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.879817:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695837.879819:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.879821:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695837.879823:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695837.879826:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5c00 x1874531139530752/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.879833:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695837.879837:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5c00 x1874531139530752/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.879844:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695837.879847:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.879849:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695837.879850:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.879852:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695837.879854:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695837.879856:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695837.879867:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695837.879870:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695837.879875:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5c00 x1874531139530752/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695837.879885:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139530752:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695837.879889:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695837.879900:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695836/real 1787695837] req@ffff8b14466f7800 x1874531139530624/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695837.879908:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695837.879910:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695837.879912:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617bf78. 00000400:00000200:0.0:1787695837.879916:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96990 00000400:00000010:0.0:1787695837.879918:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96990. 00000100:00000001:0.0:1787695837.879920:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695837.879924:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7800 x1874531139530624/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695837.879932:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7800 x1874531139530624/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.879938:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695837.879940:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.879941:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695837.879943:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695837.879947:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7800 x1874531139530624/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.879954:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695837.879959:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7800 x1874531139530624/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.879966:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695837.879968:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.879970:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695837.879971:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.879973:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695837.879975:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695837.879977:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695837.879981:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695837.879989:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695837.879995:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7800 x1874531139530624/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695837.880004:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139530624:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695837.880008:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695837.880012:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695836/real 1787695837] req@ffff8b14466f7100 x1874531139530496/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695837.880022:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695837.880023:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695837.880025:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b580. 00000400:00000200:0.0:1787695837.880028:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695837.880029:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695837.880032:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695837.880035:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7100 x1874531139530496/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695837.880043:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7100 x1874531139530496/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.880051:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695837.880052:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.880055:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695837.880056:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695837.880060:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7100 x1874531139530496/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.880068:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695837.880072:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7100 x1874531139530496/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.880080:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695837.880095:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.880097:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695837.880100:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.880102:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695837.880104:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695837.880107:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695837.880111:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695837.880113:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695837.880117:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7100 x1874531139530496/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695837.880125:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139530496:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695837.880128:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695837.880131:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695837.880134:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5880 x1874531139531008/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.880141:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695837.880142:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695837.880145:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f7400. 02000000:00000001:0.0:1787695837.880148:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695837.880150:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695837.880152:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695837.880154:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695837.880156:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:0.0:1787695837.880161:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695837.880162:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695837.880164:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5880. 00000100:00000001:0.0:1787695837.880167:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695837.880168:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.880170:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695837.880173:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6680 x1874531139530880/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.880191:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695837.880193:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695837.880195:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5800. 02000000:00000001:0.0:1787695837.880198:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695837.880199:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695837.880201:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:0.0:1787695837.880203:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695837.880205:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:0.0:1787695837.880209:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695837.880210:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695837.880211:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6680. 00000100:00000001:0.0:1787695837.880213:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695837.880215:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.880216:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695837.880232:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5c00 x1874531139530752/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.880239:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695837.880240:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695837.880242:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5c00. 02000000:00000001:0.0:1787695837.880245:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695837.880246:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695837.880247:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:0.0:1787695837.880249:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695837.880251:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675400. 02000000:00000001:0.0:1787695837.880253:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695837.880255:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695837.880256:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5c00. 00000100:00000001:0.0:1787695837.880258:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695837.880259:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.880261:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695837.880265:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7800 x1874531139530624/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.880272:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695837.880273:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695837.880283:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f5400. 02000000:00000001:0.0:1787695837.880285:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695837.880287:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695837.880289:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=15 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695837.880291:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695837.880293:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:0.0:1787695837.880295:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695837.880297:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695837.880298:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7800. 00000100:00000001:0.0:1787695837.880301:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695837.880302:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.880304:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695837.880308:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7100 x1874531139530496/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.880314:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695837.880315:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695837.880317:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b14464f4400. 02000000:00000001:0.0:1787695837.880319:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695837.880321:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695837.880322:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695837.880324:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695837.880325:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676000. 02000000:00000001:0.0:1787695837.880328:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695837.880329:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695837.880330:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7100. 00000100:00000001:0.0:1787695837.880333:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695837.880334:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.880336:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695837.880338:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695837.880340:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695837.880342:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695837.880346:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695836/real 1787695837] req@ffff8b14466f4e00 x1874531139531136/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695837.880355:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695837.880356:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695837.880404:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617be18. 00000400:00000200:0.0:1787695837.880409:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000400:00000010:0.0:1787695837.880410:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000001:0.0:1787695837.880414:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695837.880419:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4e00 x1874531139531136/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695837.880427:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4e00 x1874531139531136/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.880433:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695837.880435:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.880437:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695837.880438:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695837.880442:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4e00 x1874531139531136/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.880448:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695837.880452:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4e00 x1874531139531136/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.880458:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695837.880461:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.880463:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695837.880464:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.880465:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695837.880467:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695837.880470:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695837.880473:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695837.880475:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695837.880479:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4e00 x1874531139531136/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695837.880487:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139531136:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695837.880496:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.880498:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695837.880502:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4e00 x1874531139531136/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695861 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695837.880509:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695837.880511:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695837.880513:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b145101dc00. 02000000:00000001:0.0:1787695837.880517:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695837.880518:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695837.880520:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=16 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695837.880522:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695837.880523:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:0.0:1787695837.880526:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695837.880527:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695837.880528:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4e00. 00000100:00000001:0.0:1787695837.880531:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695837.880532:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.880534:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695837.880542:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695837.880544:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695837.880546:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695837.880547:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695838.134445:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695838.134454:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695838.134456:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695838.134463:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695838.134466:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695838.134470:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695838.134473:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695838.134475:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695838.134477:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695838.582434:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695838.582435:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695838.582447:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695838.582447:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695838.582451:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695838.582456:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695838.582457:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695838.582458:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695838.582461:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695838.582463:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695838.582464:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695838.582470:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695838.582471:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695838.582477:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695838.774443:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695838.774451:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695838.774457:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695838.774458:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695838.774462:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695838.774464:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695838.774469:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695838.774470:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695838.774474:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695838.774475:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695838.774479:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695838.774480:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695838.774540:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695838.774544:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695838.774546:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695839.158455:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695839.158462:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695839.158464:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695839.158468:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695839.158471:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695839.158474:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695839.158476:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695839.158478:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695839.158480:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695839.606444:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695839.606445:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695839.606456:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695839.606457:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695839.606459:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695839.606462:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695839.606465:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695839.606468:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695839.606469:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695839.606473:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695839.606487:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695839.606488:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695839.606495:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695839.606500:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695839.798465:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695839.798470:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695839.798481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695839.798482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695839.798485:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695839.798486:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695839.798489:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695839.798490:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695839.798492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695839.798493:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695839.798495:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695839.798495:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695839.798582:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695839.798586:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695839.798588:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695840.182462:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695840.182472:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695840.182475:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695840.182480:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695840.182484:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695840.182488:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695840.182491:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695840.182493:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695840.182496:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695840.630446:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695840.630453:0:2406:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787695840.630466:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000400:2.0:1787695840.630468:0:2406:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695755/real 1787695755] req@ffff8b1446604e00 x1874531139518336/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695840 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695840.630475:0:2405:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000400:00000001:2.0:1787695840.630491:0:2406:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000400:3.0:1787695840.630493:0:2405:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695755/real 1787695755] req@ffff8b1446604700 x1874531139518208/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695840 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695840.630495:0:2406:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695840.630499:0:2406:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14456d1318. 00000400:00000200:2.0:1787695840.630513:0:2406:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b145078a330 00000400:00000010:2.0:1787695840.630516:0:2406:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b145078a330. 00000100:00000001:2.0:1787695840.630520:0:2406:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000001:3.0:1787695840.630525:0:2405:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000200:2.0:1787695840.630526:0:2406:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446604e00 x1874531139518336/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695840 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787695840.630530:0:2405:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1787695840.630538:0:2405:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752b738. 00000400:00000200:3.0:1787695840.630549:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8110 00000400:00000010:3.0:1787695840.630555:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8110. 00000100:00000200:2.0:1787695840.630556:0:2406:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446604e00 x1874531139518336/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695840 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695840.630559:0:2405:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787695840.630566:0:2405:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446604700 x1874531139518208/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695840 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1787695840.630605:0:2405:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446604700 x1874531139518208/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695840 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695840.630607:0:2406:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695840.630611:0:2406:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695840.630620:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695840.630626:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695840.630631:0:2405:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787695840.630634:0:2405:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695840.630635:0:2406:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446604e00 x1874531139518336/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695840 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695840.630637:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695840.630643:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695840.630656:0:2406:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:2.0:1787695840.630662:0:2406:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b144515c000 already not connected (conn 22, was 11): DISCONN 00000100:00100000:3.0:1787695840.630662:0:2405:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446604700 x1874531139518208/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695840 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695840.630670:0:2406:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:2.0:1787695840.630673:0:2406:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695840.630686:0:2405:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:3.0:1787695840.630689:0:2405:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b1458081800 already not connected (conn 26, was 15): DISCONN 00000100:00000001:2.0:1787695840.630691:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695840.630694:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695840.630696:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695840.630698:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695840.630701:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695840.630703:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695840.630705:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695840.630710:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446604e00 x1874531139518336/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695840 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695840.630719:0:2405:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:3.0:1787695840.630721:0:2405:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695840.630728:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695840.630731:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695840.630739:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695840.630743:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695840.630750:0:2406:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:3.0:1787695840.630752:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695840.630761:0:2405:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695840.630762:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695840.630767:0:2405:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695840.630769:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695840.630775:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695840.630780:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446604700 x1874531139518208/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695840 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695840.630783:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446604e00 x1874531139518336/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695840 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695840.630826:0:2405:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:3.0:1787695840.630834:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695840.630835:0:2406:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446604e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:608c526c-eb6e-4a54-80a5-8969828241eb:2406:1874531139518336:192.168.201.140@tcp:400:kworker.0 00000100:00000001:3.0:1787695840.630845:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695840.630850:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695840.630851:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695840.630860:0:2406:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695840.630864:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446604700 x1874531139518208/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695840 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695840.630871:0:2406:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446604e00 x1874531139518336/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695840 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695840.630912:0:2405:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446604700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2405:1874531139518208:192.168.201.140@tcp:400:kworker.0 00000100:00000001:2.0:1787695840.630914:0:2406:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695840.630921:0:2406:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1787695840.630925:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695840.630927:0:2406:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b14474e3000. 00000100:00000001:3.0:1787695840.630934:0:2405:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1787695840.630936:0:2406:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695840.630940:0:2406:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000040:3.0:1787695840.630941:0:2405:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446604700 x1874531139518208/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695840 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1787695840.630948:0:2406:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=15 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000001:3.0:1787695840.630958:0:2405:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000020:00000001:2.0:1787695840.630959:0:2406:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787695840.630962:0:2405:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695840.630963:0:2406:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1458095100. 02000000:00000010:3.0:1787695840.630970:0:2405:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1445fe5600. 02000000:00000001:2.0:1787695840.630971:0:2406:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695840.630978:0:2406:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:3.0:1787695840.630980:0:2405:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000010:2.0:1787695840.630983:0:2406:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446604e00. 00000020:00000001:3.0:1787695840.630983:0:2405:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1787695840.630991:0:2405:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=14 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000100:00000001:2.0:1787695840.630992:0:2406:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695840.631000:0:2406:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787695840.631000:0:2405:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:2.0:1787695840.631011:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695840.631011:0:2405:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1458095300. 02000000:00000001:3.0:1787695840.631033:0:2405:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787695840.631035:0:2405:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1787695840.631045:0:2405:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446604700. 00000100:00000001:3.0:1787695840.631048:0:2405:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787695840.631049:0:2405:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695840.631053:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695840.822457:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695840.822463:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695840.822470:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695840.822472:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695840.822476:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695840.822477:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695840.822482:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695840.822482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695840.822485:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695840.822486:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695840.822505:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695840.822506:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695840.822597:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695840.822600:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695840.822603:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787695841.142484:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695841.142496:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787695841.142498:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695841.142501:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695841.142505:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695841.142516:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3728 00000400:00000200:1.0:1787695841.142524:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695841.142529:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695841.142531:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695841.142534:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695841.142536:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695841.142538:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695841.142540:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695841.142543:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695841.142545:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:1.0:1787695841.142547:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695841.142550:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:1.0:1787695841.142551:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:1.0:1787695841.142552:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:1.0:1787695841.142554:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695841.142555:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695841.142556:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695841.142558:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695841.142560:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:1.0:1787695841.142562:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695841.142564:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695841.142566:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695841.142569:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695841.142571:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695841.142590:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695841.142593:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695841.142598:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446666a00. 00000020:00000040:1.0:1787695841.142602:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:1.0:1787695841.142606:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695841.142608:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695841.142611:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695841.142614:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695841.142619:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695841.142628:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446544c00. 00000100:00000001:1.0:1787695841.142644:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695841.142654:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446666a00 x1874531139531264/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695841.142666:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695841.142671:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695841.142676:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000590093d2] to pc [ptlrpcd_rcv+-1] req@ffff8b1446666a00 x1874531139531264/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695841.142700:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695841.142705:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695841.142708:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787695841.142710:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695841.142711:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695841.142715:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695841.142717:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3728 00000400:00000200:1.0:1787695841.142722:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695841.142726:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695841.142729:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695841.142733:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695841.142735:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695841.142737:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695841.142740:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695841.142741:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695841.142747:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695841.142752:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1787695841.142753:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695841.142757:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:1.0:1787695841.142760:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695841.142763:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:1.0:1787695841.142766:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:1.0:1787695841.142768:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:0.0:1787695841.142773:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446666a00 x1874531139531264/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695841.142775:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695841.142777:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695841.142780:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695841.142784:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695841.142789:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:1.0:1787695841.142795:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695841.142801:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695841.142801:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695841.142805:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695841.142805:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695841.142811:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695841.142815:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695841.142817:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787695841.142818:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:1.0:1787695841.142821:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787695841.142822:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695841.142828:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1787695841.142828:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695841.142835:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695841.142837:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446666a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139531264:192.168.201.140@tcp:250:kworker.0 00000100:00000001:0.0:1787695841.142842:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000010:1.0:1787695841.142843:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446664380. 02000000:00000001:0.0:1787695841.142846:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:1.0:1787695841.142848:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:0.0:1787695841.142849:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695841.142854:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787695841.142856:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695841.142859:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695841.142860:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450775c00. 00000100:00000001:1.0:1787695841.142863:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695841.142864:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695841.142867:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:0.0:1787695841.142871:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:1.0:1787695841.142872:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695841.142875:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617be18. 02000000:00000010:1.0:1787695841.142881:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545000. 00000400:00000010:0.0:1787695841.142883:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00000001:1.0:1787695841.142887:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695841.142891:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139531264, portal 25 00000100:00100000:1.0:1787695841.142896:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446664380 x1874531139531392/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695841.142897:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695841.142900:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929564672 : -128555779986944 : ffff8b1446666a00) 00000100:00000001:1.0:1787695841.142909:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:0.0:1787695841.142910:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446666a00 x1874531139531264/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.142919:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787695841.142919:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695841.142924:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000040:1.0:1787695841.142928:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000094a36b85] to pc [ptlrpcd_rcv+-1] req@ffff8b1446664380 x1874531139531392/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1787695841.142929:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695841.142934:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139531264, offset 0 00000100:00000001:1.0:1787695841.142943:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695841.142944:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00080000:1.0:1787695841.142946:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695841.142950:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787695841.142951:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1787695841.142955:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:1.0:1787695841.142955:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695841.142961:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:0.0:1787695841.142965:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00080000:1.0:1787695841.142966:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3728 00000400:00000200:0.0:1787695841.142972:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695841.142977:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695841.142980:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695841.142982:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695841.142985:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16960:16960:256:4294967295] 192.168.201.140@tcp LPNI seq info [16959:16959:8:4294967295] 00000400:00000200:0.0:1787695841.142988:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695841.142993:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695841.143000:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695841.143002:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695841.143004:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695841.143007:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1787695841.143011:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000040:1.0:1787695841.143013:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695841.143016:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695841.143017:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695841.143018:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:1.0:1787695841.143019:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695841.143022:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:1.0:1787695841.143026:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:1.0:1787695841.143028:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000800:00000200:0.0:1787695841.143031:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000040:1.0:1787695841.143033:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695841.143036:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:0.0:1787695841.143037:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfcd00. 00000100:00000040:1.0:1787695841.143039:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695841.143042:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695841.143044:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000800:00000200:0.0:1787695841.143048:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00080000:1.0:1787695841.143049:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000800:00000200:0.0:1787695841.143053:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 29 secs later) 00000800:00000200:0.1:1787695841.143061:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:1.0:1787695841.143062:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1787695841.143064:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 29 secs later) 02000000:00000001:1.0:1787695841.143067:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:0.0:1787695841.143071:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695841.143074:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfcd00. 02000000:00000001:1.0:1787695841.143075:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1787695841.143079:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:1.0:1787695841.143080:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695841.143085:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:0.0:1787695841.143086:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695841.143091:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695841.143093:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695841.143095:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695841.143100:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446666a00 x1874531139531264/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787695841.143102:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787695841.143111:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695841.143113:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787695841.143115:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695841.143118:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446666a00 x1874531139531264/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695866 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1787695841.143120:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446664e00. 00000020:00000040:1.0:1787695841.143126:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=15 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000100:00000001:0.0:1787695841.143127:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695841.143130:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787695841.143131:0:2402:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1787695841.143132:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695841.143135:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787695841.143137:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695841.143137:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695841.143139:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695841.143143:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000010:1.0:1787695841.143146:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446544400. 00000100:00000001:0.0:1787695841.143147:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298412856 00000100:00000001:1.0:1787695841.143152:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695841.143155:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298407736 00000100:00000001:0.0:1787695841.143173:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298412856 00000100:00100000:1.0:1787695841.143175:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446664e00 x1874531139531520/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695841.143178:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298407736 00000100:00000001:0.0:1787695841.143185:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695841.143188:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695841.143190:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695841.143191:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695841.143198:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:1.0:1787695841.143199:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b012f68f] to pc [ptlrpcd_rcv+-1] req@ffff8b1446664e00 x1874531139531520/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695841.143208:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695841.143209:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695841/real 1787695841] req@ffff8b1446666a00 x1874531139531264/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695841.143211:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695841.143218:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787695841.143220:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695841.143223:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695841.143228:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695841.143231:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3728 00000400:00000200:1.0:1787695841.143234:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000001:0.0:1787695841.143245:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:1.0:1787695841.143246:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000001:0.0:1787695841.143249:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:1.0:1787695841.143250:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000400:00000010:0.0:1787695841.143252:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617be18. 00000100:00000040:1.0:1787695841.143256:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695841.143258:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000100:00000001:1.0:1787695841.143259:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:0.0:1787695841.143263:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000001:0.0:1787695841.143266:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695841.143270:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446666a00 x1874531139531264/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695841.143272:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695841.143280:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000200:0.0:1787695841.143283:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446666a00 x1874531139531264/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787695841.143284:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695841.143287:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:1.0:1787695841.143291:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787695841.143296:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:1.0:1787695841.143296:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000100:00000001:0.0:1787695841.143299:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.143303:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:1.0:1787695841.143303:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:0.0:1787695841.143306:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787695841.143310:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00100000:0.0:1787695841.143311:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446666a00 x1874531139531264/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695841.143321:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695841.143322:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695841.143327:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695841.143329:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446666a00 x1874531139531264/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695841.143334:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695841.143340:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695841.143342:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695841.143344:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.143348:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:1.0:1787695841.143349:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00000001:0.0:1787695841.143350:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.143355:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:1.0:1787695841.143357:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695841.143358:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695841.143361:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695841.143400:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695841.143403:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695841.143406:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695841.143407:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1787695841.143409:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00080000:0.0:1787695841.143413:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000008:00100000:1.0:1787695841.143415:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787695841.143416:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00000001:1.0:1787695841.143423:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695841.143424:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446666a00 x1874531139531264/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695841.143428:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695841.143435:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446666300. 00000100:00100000:0.0:1787695841.143441:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446666a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139531264:192.168.201.140@tcp:250:kworker.0 00000020:00000040:1.0:1787695841.143442:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000100:00000001:0.0:1787695841.143446:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787695841.143448:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787695841.143449:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1787695841.143452:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695841.143454:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446666a00 x1874531139531264/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695841.143457:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787695841.143461:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695841.143463:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695841.143465:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695841.143467:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:1.0:1787695841.143469:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:0.0:1787695841.143470:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695841.143473:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 02000000:00000001:1.0:1787695841.143474:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695841.143477:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695841.143480:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446546000. 02000000:00000010:0.0:1787695841.143481:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446544c00. 00000100:00000001:1.0:1787695841.143487:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695841.143488:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695841.143491:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695841.143493:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446666a00. 00000100:00100000:1.0:1787695841.143496:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446666300 x1874531139531648/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695841.143497:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695841.143500:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.143505:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695841.143511:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695841.143513:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695841.143513:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695841.143518:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695841.143518:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000945f4e20] to pc [ptlrpcd_rcv+-1] req@ffff8b1446666300 x1874531139531648/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.143523:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695841.143525:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695841.143527:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787695841.143534:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695841.143537:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695841.143539:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1787695841.143540:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:0.0:1787695841.143541:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446664380 x1874531139531392/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695841.143542:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695841.143546:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695841.143549:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3728 00000400:00000200:1.0:1787695841.143555:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695841.143560:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695841.143565:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787695841.143567:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1787695841.143569:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695841.143571:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695841.143572:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695841.143576:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695841.143578:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1787695841.143581:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:0.0:1787695841.143582:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1787695841.143587:0:38:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:0.0:1787695841.143588:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1787695841.143590:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000100:00100000:0.0:1787695841.143596:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446664380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139531392:192.168.201.140@tcp:38:kworker.0 00000020:00000001:1.0:1787695841.143596:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695841.143600:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:1.0:1787695841.143604:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000001:0.0:1787695841.143607:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:1.0:1787695841.143609:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 02000000:00000001:0.0:1787695841.143610:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695841.143612:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695841.143613:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695841.143616:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695841.143618:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695841.143626:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:1.0:1787695841.143626:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000010:0.0:1787695841.143631:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617be18. 00000100:00000001:1.0:1787695841.143632:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:0.0:1787695841.143636:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00000040:1.0:1787695841.143637:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000200:0.0:1787695841.143641:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139531392, portal 10 00000100:00000001:1.0:1787695841.143641:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695841.143643:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00000001:0.0:1787695841.143646:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695841.143649:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929554816 : -128555779996800 : ffff8b1446664380) 00000100:00080000:1.0:1787695841.143649:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695841.143654:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695841.143656:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695841.143659:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1787695841.143660:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446664380 x1874531139531392/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695841.143664:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695841.143669:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787695841.143672:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787695841.143678:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:1.0:1787695841.143678:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695841.143682:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695841.143686:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000010:1.0:1787695841.143687:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446666a00. 00000020:00000040:1.0:1787695841.143691:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 00000400:00000010:0.0:1787695841.143692:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 02000000:00000001:1.0:1787695841.143693:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695841.143698:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695841.143699:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139531392, offset 0 00000100:00000001:1.0:1787695841.143700:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695841.143702:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695841.143705:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695841.143710:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 02000000:00000010:1.0:1787695841.143710:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446544000. 00000100:00000001:1.0:1787695841.143716:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695841.143721:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00100000:1.0:1787695841.143724:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446666a00 x1874531139531776/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1787695841.143730:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:1.0:1787695841.143735:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1787695841.143738:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1787695841.143738:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695841.143743:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000590093d2] to pc [ptlrpcd_rcv+-1] req@ffff8b1446666a00 x1874531139531776/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695841.143747:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695841.143754:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1787695841.143757:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695841.143761:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00080000:1.0:1787695841.143761:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1787695841.143766:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16961:16961:256:4294967295] 192.168.201.140@tcp LPNI seq info [16960:16960:8:4294967295] 00000100:00080000:1.0:1787695841.143770:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787695841.143774:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1787695841.143776:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:1.0:1787695841.143777:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695841.143782:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695841.143789:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3728 00000400:00000200:0.0:1787695841.143793:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000400:00000200:1.0:1787695841.143798:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000800:00000200:0.0:1787695841.143802:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695841.143806:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000800:00000010:0.0:1787695841.143808:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfcd00. 00000100:00080000:1.0:1787695841.143809:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695841.143815:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000800:00000200:0.0:1787695841.143819:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:1.0:1787695841.143819:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695841.143822:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695841.143824:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000200:0.0:1787695841.143831:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 29 secs later) 00000020:00000001:1.0:1787695841.143831:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695841.143833:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:1.0:1787695841.143839:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000200:0.1:1787695841.143839:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695841.143844:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 29 secs later) 00000020:00000040:1.0:1787695841.143845:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000800:00000100:0.0:1787695841.143849:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695841.143852:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfcd00. 00000020:00000001:1.0:1787695841.143852:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000200:0.0:1787695841.143857:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:1.0:1787695841.143858:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000200:0.0:1787695841.143862:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695841.143867:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000100:00000040:1.0:1787695841.143867:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000010:0.0:1787695841.143869:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695841.143874:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787695841.143874:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:0.0:1787695841.143883:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446664380 x1874531139531392/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695841.143883:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695841.143887:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695841.143893:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:1.0:1787695841.143899:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695841.143900:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787695841.143902:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695841.143906:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:0.0:1787695841.143907:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446664380 x1874531139531392/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695866 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695841.143909:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695841.143914:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695841.143919:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787695841.143922:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1787695841.143923:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787695841.143925:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695841.143928:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695841.143933:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000008:00000001:1.0:1787695841.143934:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695841.143939:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695841.143942:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000010:1.0:1787695841.143945:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665c00. 00000100:00000001:0.0:1787695841.143946:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298402758 00000020:00000040:1.0:1787695841.143950:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000100:00000001:0.0:1787695841.143951:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298412856 02000000:00000001:1.0:1787695841.143955:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787695841.143956:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298402758 02000000:00000001:1.0:1787695841.143959:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695841.143960:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298412856 00000100:00000001:1.0:1787695841.143963:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787695841.143964:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695841.143966:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787695841.143967:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695841.143969:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695841.143971:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000010:1.0:1787695841.143974:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446547800. 00000100:00100000:0.0:1787695841.143975:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695841/real 1787695841] req@ffff8b1446664380 x1874531139531392/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695841.143980:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787695841.143987:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695841.143990:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:1.0:1787695841.143990:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446665c00 x1874531139531904/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:0.0:1787695841.143998:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617be18. 00000100:00000001:1.0:1787695841.143999:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1787695841.144004:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000100:00000001:1.0:1787695841.144004:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695841.144008:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000040:1.0:1787695841.144009:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d782cc4] to pc [ptlrpcd_rcv+-1] req@ffff8b1446665c00 x1874531139531904/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.144011:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695841.144018:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446664380 x1874531139531392/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695841.144020:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695841.144025:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695841.144031:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000200:0.0:1787695841.144032:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446664380 x1874531139531392/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695841.144033:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695841.144036:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695841.144042:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787695841.144046:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:1.0:1787695841.144046:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3728 00000100:00000001:0.0:1787695841.144049:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.144052:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695841.144055:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1787695841.144055:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00100000:0.0:1787695841.144061:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446664380 x1874531139531392/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695841.144064:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:0.0:1787695841.144072:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787695841.144072:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1787695841.144077:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446664380 x1874531139531392/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695841.144082:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695841.144086:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695841.144087:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695841.144089:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.144092:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695841.144095:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695841.144096:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695841.144097:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695841.144098:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695841.144100:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695841.144104:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695841.144108:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1787695841.144108:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695841.144114:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000040:0.0:1787695841.144116:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446664380 x1874531139531392/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1787695841.144117:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000020:00000001:1.0:1787695841.144120:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695841.144124:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:1.0:1787695841.144127:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00100000:0.0:1787695841.144129:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446664380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139531392:192.168.201.140@tcp:38:kworker.0 00000020:00000001:1.0:1787695841.144130:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:1.0:1787695841.144133:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695841.144134:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695841.144138:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695841.144142:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695841.144143:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:1.0:1787695841.144145:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00000040:0.0:1787695841.144150:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446664e00 x1874531139531520/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695841.144151:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695841.144156:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695841.144160:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695841.144175:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695841.144179:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1787695841.144179:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695841.144183:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695841.144184:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787695841.144186:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787695841.144190:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:1.0:1787695841.144190:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695841.144195:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695841.144196:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:1.0:1787695841.144198:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446666d80. 02000000:00000001:0.0:1787695841.144204:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1787695841.144204:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=16 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:1.0:1787695841.144209:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695841.144211:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695841.144214:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446664e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139531520:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695841.144215:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695841.144217:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695841.144219:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695841.144224:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000010:1.0:1787695841.144225:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446547c00. 02000000:00000001:0.0:1787695841.144227:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695841.144230:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695841.144230:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695841.144234:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:1.0:1787695841.144238:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446666d80 x1874531139532032/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:0.0:1787695841.144239:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695841.144244:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695841.144253:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695841.144255:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617be18. 00000400:00000010:0.0:1787695841.144259:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00000200:0.0:1787695841.144264:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139531520, portal 10 00000100:00000001:1.0:1787695841.144266:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695841.144268:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695841.144271:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929557504 : -128555779994112 : ffff8b1446664e00) 00000100:00000001:1.0:1787695841.144271:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695841.144279:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa82a36] to pc [ptlrpcd_rcv+-1] req@ffff8b1446666d80 x1874531139532032/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695841.144282:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446664e00 x1874531139531520/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.144303:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787695841.144304:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695841.144305:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695841.144309:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000040:1.0:1787695841.144311:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:0.0:1787695841.144312:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139531520, offset 0 00000400:00000200:0.0:1787695841.144319:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:1.0:1787695841.144321:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:0.0:1787695841.144324:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:1.0:1787695841.144328:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1787695841.144329:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695841.144335:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:1.0:1787695841.144337:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:1.0:1787695841.144342:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3738) 00000400:00000200:0.0:1787695841.144345:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695841.144349:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695841.144350:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695841.144353:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16962:16962:256:4294967295] 192.168.201.140@tcp LPNI seq info [16961:16961:8:4294967295] 00000400:00000200:0.0:1787695841.144357:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695841.144380:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695841.144385:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695841.144388:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfcd00. 00000800:00000200:0.0:1787695841.144391:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695841.144394:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 29 secs later) 00000800:00000200:0.1:1787695841.144397:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695841.144399:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 29 secs later) 00000800:00000100:0.0:1787695841.144401:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695841.144403:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfcd00. 00000400:00000200:0.0:1787695841.144405:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695841.144408:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695841.144410:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695841.144412:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695841.144413:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695841.144416:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446664e00 x1874531139531520/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.144422:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695841.144424:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446664e00 x1874531139531520/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695866 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.144428:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695841.144430:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695841.144432:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695841.144434:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:0.0:1787695841.144436:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695841.144438:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298401607 00000100:00000001:0.0:1787695841.144439:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298412856 00000100:00000001:0.0:1787695841.144441:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298401607 00000100:00000001:0.0:1787695841.144447:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298412856 00000100:00000001:0.0:1787695841.144448:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695841.144450:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695841.144451:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695841.144454:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695841/real 1787695841] req@ffff8b1446664e00 x1874531139531520/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695841.144459:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695841.144460:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695841.144461:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617be18. 00000400:00000200:0.0:1787695841.144463:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000400:00000010:0.0:1787695841.144464:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000001:0.0:1787695841.144466:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695841.144468:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446664e00 x1874531139531520/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695841.144473:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446664e00 x1874531139531520/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.144477:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695841.144478:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.144479:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695841.144480:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695841.144482:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446664e00 x1874531139531520/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.144486:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695841.144488:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446664e00 x1874531139531520/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.144492:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695841.144494:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.144495:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695841.144496:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.144501:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695841.144502:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695841.144504:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695841.144506:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695841.144507:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695841.144510:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446664e00 x1874531139531520/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695841.144515:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446664e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139531520:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695841.144517:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.144519:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695841.144521:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446664380 x1874531139531392/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.144525:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695841.144526:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695841.144527:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695841.144529:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695841.144530:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695841.144531:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695841.144532:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695841.144533:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545000. 02000000:00000001:0.0:1787695841.144536:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695841.144538:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695841.144539:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446664380. 00000100:00000001:0.0:1787695841.144542:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695841.144543:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.144545:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695841.144548:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446664e00 x1874531139531520/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.144555:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695841.144557:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695841.144558:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695841.144566:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695841.144568:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695841.144569:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=14 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695841.144572:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695841.144574:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446544400. 02000000:00000001:0.0:1787695841.144577:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695841.144578:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695841.144580:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446664e00. 00000100:00000001:0.0:1787695841.144582:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695841.144584:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.144586:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.144590:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695841.144591:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695841.144594:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695841.144595:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695841.144596:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695841.144599:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446666300 x1874531139531648/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.144604:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695841.144605:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695841.144607:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695841.144608:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695841.144610:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695841.144611:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446666300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139531648:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695841.144614:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695841.144615:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695841.144616:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695841.144617:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695841.144620:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695841.144621:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695841.144624:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695841.144625:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617be18. 00000400:00000010:0.0:1787695841.144628:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00000200:0.0:1787695841.144630:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139531648, portal 4 00000100:00000001:0.0:1787695841.144631:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695841.144636:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929562880 : -128555779988736 : ffff8b1446666300) 00000100:00000040:0.0:1787695841.144640:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446666300 x1874531139531648/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.144645:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695841.144647:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695841.144648:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695841.144650:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139531648, offset 0 00000400:00000200:0.0:1787695841.144652:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695841.144655:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695841.144657:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695841.144659:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695841.144661:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695841.144663:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695841.144665:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695841.144668:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16963:16963:256:4294967295] 192.168.201.140@tcp LPNI seq info [16962:16962:8:4294967295] 00000400:00000200:0.0:1787695841.144672:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695841.144678:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695841.144683:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695841.144686:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfcd00. 00000800:00000200:0.0:1787695841.144690:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695841.144694:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 29 secs later) 00000800:00000200:0.1:1787695841.144698:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695841.144701:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 29 secs later) 00000800:00000100:0.0:1787695841.144705:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695841.144707:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfcd00. 00000400:00000200:0.0:1787695841.144711:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695841.144714:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695841.144718:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695841.144720:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695841.144722:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695841.144726:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446666300 x1874531139531648/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.144734:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695841.144746:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446666300 x1874531139531648/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695866 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.144753:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695841.144755:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695841.144759:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695841.144763:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:0.0:1787695841.144766:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695841.144769:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298401637 00000100:00000001:0.0:1787695841.144771:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298412856 00000100:00000001:0.0:1787695841.144773:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298401637 00000100:00000001:0.0:1787695841.144775:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298412856 00000100:00000001:0.0:1787695841.144777:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695841.144778:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695841.144780:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695841.144783:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695841/real 1787695841] req@ffff8b1446666300 x1874531139531648/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695841.144788:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695841.144789:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695841.144791:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617be18. 00000400:00000200:0.0:1787695841.144793:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000400:00000010:0.0:1787695841.144794:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000001:0.0:1787695841.144795:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695841.144797:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446666300 x1874531139531648/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695841.144803:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446666300 x1874531139531648/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.144807:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695841.144808:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.144809:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695841.144810:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695841.144816:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446666300 x1874531139531648/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.144821:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695841.144824:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446666300 x1874531139531648/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.144828:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695841.144829:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.144830:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695841.144831:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.144832:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695841.144834:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695841.144835:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695841.144837:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695841.144838:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695841.144841:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446666300 x1874531139531648/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695841.144845:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446666300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139531648:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695841.144848:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695841.144850:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446666a00 x1874531139531776/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.144854:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695841.144855:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695841.144857:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695841.144859:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695841.144860:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695841.144861:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446666a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139531776:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695841.144863:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695841.144867:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695841.144868:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695841.144870:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695841.144871:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695841.144873:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695841.144875:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695841.144877:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617be18. 00000400:00000010:0.0:1787695841.144879:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00000200:0.0:1787695841.144881:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139531776, portal 4 00000100:00000001:0.0:1787695841.144883:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695841.144884:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929564672 : -128555779986944 : ffff8b1446666a00) 00000100:00000040:0.0:1787695841.144888:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446666a00 x1874531139531776/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.144892:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695841.144894:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695841.144895:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695841.144897:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139531776, offset 0 00000400:00000200:0.0:1787695841.144899:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695841.144902:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695841.144904:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695841.144906:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695841.144908:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695841.144909:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695841.144911:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695841.144913:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16964:16964:256:4294967295] 192.168.201.140@tcp LPNI seq info [16963:16963:8:4294967295] 00000400:00000200:0.0:1787695841.144916:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695841.144920:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695841.144923:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695841.144926:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfcd00. 00000800:00000200:0.0:1787695841.144928:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695841.144930:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 29 secs later) 00000800:00000200:0.1:1787695841.144933:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695841.144935:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 29 secs later) 00000800:00000100:0.0:1787695841.144937:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695841.144942:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfcd00. 00000400:00000200:0.0:1787695841.144944:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695841.144947:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695841.144949:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695841.144950:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695841.144952:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695841.144954:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446666a00 x1874531139531776/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.144960:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695841.144962:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446666a00 x1874531139531776/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695866 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.144966:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695841.144967:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695841.144970:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695841.144972:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:0.0:1787695841.144974:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695841.144975:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298401309 00000100:00000001:0.0:1787695841.144977:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298412856 00000100:00000001:0.0:1787695841.144978:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298401309 00000100:00000001:0.0:1787695841.144980:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298412856 00000100:00000001:0.0:1787695841.144981:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695841.144982:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695841.144984:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695841.144986:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695841/real 1787695841] req@ffff8b1446666a00 x1874531139531776/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695841.144990:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695841.144991:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695841.144992:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617be18. 00000400:00000200:0.0:1787695841.144993:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000400:00000010:0.0:1787695841.144994:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000001:0.0:1787695841.144999:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695841.145001:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446666a00 x1874531139531776/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695841.145007:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446666a00 x1874531139531776/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.145010:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695841.145011:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.145012:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695841.145013:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695841.145015:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446666a00 x1874531139531776/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.145019:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695841.145022:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446666a00 x1874531139531776/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.145026:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695841.145028:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.145029:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695841.145030:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.145031:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695841.145032:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695841.145033:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695841.145035:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695841.145036:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695841.145039:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446666a00 x1874531139531776/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695841.145043:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446666a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139531776:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695841.145046:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695841.145048:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665c00 x1874531139531904/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.145055:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695841.145056:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695841.145057:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695841.145059:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695841.145060:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695841.145062:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139531904:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695841.145063:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695841.145065:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695841.145066:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695841.145067:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695841.145069:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450777000. 02000000:00000001:0.0:1787695841.145071:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695841.145073:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695841.145074:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617be18. 00000400:00000010:0.0:1787695841.145077:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00000200:0.0:1787695841.145078:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139531904, portal 10 00000100:00000001:0.0:1787695841.145080:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695841.145081:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929561088 : -128555779990528 : ffff8b1446665c00) 00000100:00000040:0.0:1787695841.145085:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665c00 x1874531139531904/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.145089:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695841.145091:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695841.145092:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695841.145094:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139531904, offset 0 00000400:00000200:0.0:1787695841.145096:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695841.145098:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695841.145100:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695841.145102:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695841.145103:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695841.145105:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695841.145106:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695841.145111:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16965:16965:256:4294967295] 192.168.201.140@tcp LPNI seq info [16964:16964:8:4294967295] 00000400:00000200:0.0:1787695841.145114:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695841.145119:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695841.145122:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695841.145124:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfcd00. 00000800:00000200:0.0:1787695841.145127:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695841.145129:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 29 secs later) 00000800:00000200:0.1:1787695841.145131:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695841.145133:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 29 secs later) 00000800:00000100:0.0:1787695841.145140:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695841.145141:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfcd00. 00000400:00000200:0.0:1787695841.145143:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695841.145146:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695841.145148:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695841.145149:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695841.145151:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695841.145153:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446665c00 x1874531139531904/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.145159:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695841.145175:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665c00 x1874531139531904/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695866 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.145181:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695841.145183:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695841.145187:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695841.145190:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:0.0:1787695841.145193:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695841.145196:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298401495 00000100:00000001:0.0:1787695841.145198:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298412856 00000100:00000001:0.0:1787695841.145200:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298401495 00000100:00000001:0.0:1787695841.145203:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298412856 00000100:00000001:0.0:1787695841.145205:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695841.145213:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695841.145215:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695841.145220:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695841/real 1787695841] req@ffff8b1446665c00 x1874531139531904/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695841.145228:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695841.145229:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695841.145231:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617be18. 00000400:00000200:0.0:1787695841.145233:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000400:00000010:0.0:1787695841.145234:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000001:0.0:1787695841.145235:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695841.145237:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665c00 x1874531139531904/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695841.145243:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665c00 x1874531139531904/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.145247:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695841.145248:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.145249:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695841.145250:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695841.145252:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446665c00 x1874531139531904/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.145256:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695841.145259:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665c00 x1874531139531904/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.145264:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695841.145265:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.145266:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695841.145267:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.145268:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695841.145269:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695841.145271:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695841.145277:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695841.145278:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695841.145281:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665c00 x1874531139531904/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695841.145286:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139531904:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695841.145288:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695841.145290:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446666d80 x1874531139532032/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.145294:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695841.145295:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695841.145297:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695841.145298:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695841.145299:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695841.145301:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446666d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139532032:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695841.145302:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695841.145304:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695841.145305:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695841.145306:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695841.145308:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450775000. 02000000:00000001:0.0:1787695841.145309:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695841.145312:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695841.145314:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617be18. 00000400:00000010:0.0:1787695841.145316:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00000200:0.0:1787695841.145318:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139532032, portal 10 00000100:00000001:0.0:1787695841.145319:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695841.145320:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929565568 : -128555779986048 : ffff8b1446666d80) 00000100:00000040:0.0:1787695841.145324:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446666d80 x1874531139532032/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.145328:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695841.145332:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695841.145333:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695841.145336:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139532032, offset 0 00000400:00000200:0.0:1787695841.145338:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695841.145341:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695841.145344:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695841.145346:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695841.145347:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695841.145349:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695841.145351:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695841.145353:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16966:16966:256:4294967295] 192.168.201.140@tcp LPNI seq info [16965:16965:8:4294967295] 00000400:00000200:0.0:1787695841.145355:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695841.145360:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695841.145379:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695841.145381:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfcd00. 00000800:00000200:0.0:1787695841.145385:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695841.145387:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 29 secs later) 00000800:00000200:0.1:1787695841.145391:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695841.145393:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 29 secs later) 00000800:00000100:0.0:1787695841.145397:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695841.145398:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfcd00. 00000400:00000200:0.0:1787695841.145402:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695841.145404:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695841.145406:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695841.145408:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695841.145410:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695841.145414:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446666d80 x1874531139532032/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695866 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.145421:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695841.145425:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446666d80 x1874531139532032/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695866 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.145430:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695841.145431:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695841.145437:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695841.145440:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:0.0:1787695841.145442:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695841.145443:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298401474 00000100:00000001:0.0:1787695841.145445:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298412856 00000100:00000001:0.0:1787695841.145447:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298401474 00000100:00000001:0.0:1787695841.145448:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298412856 00000100:00000001:0.0:1787695841.145450:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695841.145451:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695841.145453:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695841.145455:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695841/real 1787695841] req@ffff8b1446666d80 x1874531139532032/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695841.145460:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695841.145461:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695841.145462:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617be18. 00000400:00000200:0.0:1787695841.145463:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000400:00000010:0.0:1787695841.145464:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000001:0.0:1787695841.145465:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695841.145467:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446666d80 x1874531139532032/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695841.145472:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446666d80 x1874531139532032/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.145476:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695841.145477:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.145479:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695841.145480:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695841.145482:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446666d80 x1874531139532032/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.145486:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695841.145488:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446666d80 x1874531139532032/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.145496:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695841.145497:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.145498:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695841.145499:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.145500:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695841.145501:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695841.145503:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695841.145505:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695841.145506:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695841.145508:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446666d80 x1874531139532032/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695841.145513:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446666d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139532032:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695841.145516:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.145517:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695841.145519:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446666300 x1874531139531648/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.145523:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695841.145524:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695841.145526:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695841.145527:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695841.145528:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695841.145529:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:0.0:1787695841.145530:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695841.145531:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446546000. 02000000:00000001:0.0:1787695841.145535:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695841.145536:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695841.145537:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446666300. 00000100:00000001:0.0:1787695841.145540:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695841.145541:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.145547:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695841.145551:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446666a00 x1874531139531776/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.145557:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695841.145559:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695841.145560:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695841.145563:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695841.145564:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695841.145565:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:0.0:1787695841.145567:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695841.145569:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446544000. 02000000:00000001:0.0:1787695841.145571:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695841.145573:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695841.145574:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446666a00. 00000100:00000001:0.0:1787695841.145576:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695841.145578:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.145580:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695841.145583:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665c00 x1874531139531904/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.145590:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695841.145592:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695841.145593:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450777000. 02000000:00000001:0.0:1787695841.145595:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695841.145596:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695841.145597:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695841.145598:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695841.145599:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446547800. 02000000:00000001:0.0:1787695841.145601:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695841.145602:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695841.145603:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665c00. 00000100:00000001:0.0:1787695841.145604:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695841.145604:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.145605:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695841.145608:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446666d80 x1874531139532032/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695866 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695841.145623:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695841.145624:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695841.145626:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450775000. 02000000:00000001:0.0:1787695841.145628:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695841.145629:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695841.145631:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=15 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695841.145633:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695841.145634:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446547c00. 02000000:00000001:0.0:1787695841.145637:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695841.145638:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695841.145639:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446666d80. 00000100:00000001:0.0:1787695841.145642:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695841.145643:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.145645:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695841.145649:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695841.145651:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695841.145653:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695841.145654:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695841.206517:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695841.206526:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695841.206528:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695841.206533:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695841.206537:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695841.206541:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695841.206543:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695841.206546:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695841.206548:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695841.654491:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695841.654506:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695841.654508:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695841.654514:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695841.654516:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695841.654520:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695841.654547:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695841.654574:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695841.654579:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695841.654580:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695841.654581:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695841.654582:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695841.654585:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695841.654586:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695841.846474:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695841.846479:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695841.846484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695841.846485:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695841.846487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695841.846488:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695841.846491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695841.846492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695841.846494:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695841.846496:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695841.846498:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695841.846503:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695841.846504:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695841.846506:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695841.846508:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695841.846510:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695841.846511:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695841.846512:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695841.846515:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695841.846516:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695841.846574:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695841.846579:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695841.846583:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695842.230452:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695842.230458:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695842.230460:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695842.230464:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695842.230467:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695842.230470:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695842.230472:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695842.230474:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695842.230476:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695842.678421:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695842.678432:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695842.678433:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695842.678438:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695842.678440:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695842.678444:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695842.678447:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695842.678479:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695842.678484:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695842.678486:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695842.678488:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695842.678489:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695842.678492:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695842.678494:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695842.870439:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695842.870460:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695842.870465:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695842.870466:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695842.870468:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695842.870469:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695842.870472:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695842.870472:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695842.870475:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695842.870476:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695842.870478:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695842.870479:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695842.870544:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695842.870547:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695842.870550:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695843.254470:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695843.254476:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695843.254478:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695843.254483:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695843.254485:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695843.254489:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695843.254490:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695843.254509:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695843.254511:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695843.702463:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695843.702463:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695843.702475:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695843.702476:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695843.702479:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695843.702480:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695843.702485:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695843.702487:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695843.702488:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695843.702490:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695843.702493:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695843.702496:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695843.702497:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695843.702504:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695843.894561:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695843.894572:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695843.894574:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695843.894578:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695843.894583:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695843.894586:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695843.894588:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695843.894591:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695843.894596:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695843.894598:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695843.894599:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695843.894601:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695843.894602:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695843.894604:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695843.894606:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695843.894608:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695843.894613:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695843.894615:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695843.894619:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695843.894621:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695843.894625:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695843.894627:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695843.894631:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695843.894633:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695843.894702:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695843.894706:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695843.894709:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695844.278448:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695844.278455:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695844.278457:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695844.278461:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695844.278464:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695844.278466:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695844.278468:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695844.278470:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695844.278472:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695844.726428:0:2403:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695844.726433:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695844.726434:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695844.726439:0:2406:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787695844.726440:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695844.726440:0:2405:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787695844.726444:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695844.726450:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695844.726453:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695844.726457:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000400:2.0:1787695844.726460:0:2405:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695759/real 1787695759] req@ffff8b1446606680 x1874531139518592/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695844 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:3.0:1787695844.726460:0:2406:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695759/real 1787695759] req@ffff8b1446606d80 x1874531139518848/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695844 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695844.726461:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787695844.726499:0:2406:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787695844.726502:0:2406:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1787695844.726504:0:2406:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14456d1738. 00000400:00000200:3.0:1787695844.726509:0:2406:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b145078a440 00000400:00000010:3.0:1787695844.726511:0:2406:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b145078a440. 00000100:00000001:3.0:1787695844.726514:0:2406:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000001:2.0:1787695844.726515:0:2405:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695844.726519:0:2405:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000200:3.0:1787695844.726519:0:2406:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446606d80 x1874531139518848/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695844 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1787695844.726522:0:2405:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752b160. 00000400:00000200:2.0:1787695844.726529:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8440 00000400:00000010:2.0:1787695844.726534:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8440. 00000100:00000200:3.0:1787695844.726534:0:2406:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446606d80 x1874531139518848/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695844 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695844.726540:0:2405:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787695844.726546:0:2406:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000200:2.0:1787695844.726547:0:2405:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446606680 x1874531139518592/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695844 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695844.726549:0:2406:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695844.726554:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000200:2.0:1787695844.726559:0:2405:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446606680 x1874531139518592/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695844 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695844.726573:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787695844.726582:0:2406:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446606d80 x1874531139518848/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695844 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695844.726583:0:2405:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695844.726586:0:2405:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695844.726591:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695844.726594:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695844.726597:0:2406:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:3.0:1787695844.726601:0:2406:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b144515c000 already not connected (conn 23, was 11): DISCONN 00000100:00100000:2.0:1787695844.726602:0:2405:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446606680 x1874531139518592/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695844 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695844.726606:0:2406:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:3.0:1787695844.726609:0:2406:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695844.726615:0:2405:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00000001:3.0:1787695844.726616:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1787695844.726618:0:2405:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b1458081800 already not connected (conn 27, was 15): DISCONN 00000100:00000001:3.0:1787695844.726619:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695844.726622:0:2405:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:3.0:1787695844.726623:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695844.726625:0:2405:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695844.726626:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695844.726630:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695844.726631:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695844.726632:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695844.726634:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695844.726635:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695844.726639:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695844.726642:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1787695844.726643:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446606d80 x1874531139518848/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695844 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695844.726645:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695844.726650:0:2405:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695844.726652:0:2405:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695844.726658:0:2406:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000040:2.0:1787695844.726662:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446606680 x1874531139518592/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695844 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695844.726663:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695844.726668:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695844.726671:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695844.726677:0:2405:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000040:3.0:1787695844.726679:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446606d80 x1874531139518848/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695844 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695844.726682:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695844.726687:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695844.726690:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695844.726698:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446606680 x1874531139518592/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695844 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695844.726699:0:2406:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446606d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:608c526c-eb6e-4a54-80a5-8969828241eb:2406:1874531139518848:192.168.201.140@tcp:400:kworker.0 00000100:00000001:3.0:1787695844.726709:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695844.726713:0:2406:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1787695844.726715:0:2405:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446606680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2405:1874531139518592:192.168.201.140@tcp:400:kworker.0 00000100:00000040:3.0:1787695844.726720:0:2406:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446606d80 x1874531139518848/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695844 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695844.726722:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695844.726727:0:2405:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787695844.726734:0:2406:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787695844.726736:0:2406:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1787695844.726740:0:2406:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b14474e3e00. 00000100:00000040:2.0:1787695844.726741:0:2405:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446606680 x1874531139518592/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695844 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695844.726748:0:2406:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787695844.726760:0:2406:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:2.0:1787695844.726764:0:2405:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000020:00000040:3.0:1787695844.726765:0:2406:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=14 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:2.0:1787695844.726767:0:2405:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695844.726771:0:2405:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1445fe4600. 00000020:00000001:3.0:1787695844.726772:0:2406:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787695844.726776:0:2405:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000010:3.0:1787695844.726778:0:2406:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1458095e00. 00000020:00000001:2.0:1787695844.726779:0:2405:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695844.726781:0:2405:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=13 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:2.0:1787695844.726785:0:2405:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695844.726787:0:2405:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1458095f00. 02000000:00000001:3.0:1787695844.726787:0:2406:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787695844.726791:0:2406:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1787695844.726793:0:2405:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695844.726796:0:2405:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1787695844.726797:0:2406:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446606d80. 00000100:00000010:2.0:1787695844.726798:0:2405:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446606680. 00000100:00000001:2.0:1787695844.726804:0:2405:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787695844.726806:0:2406:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695844.726807:0:2405:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695844.726812:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695844.726812:0:2406:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695844.726819:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695844.918524:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695844.918531:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695844.918536:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695844.918563:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695844.918566:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695844.918567:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695844.918571:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695844.918572:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695844.918574:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695844.918575:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695844.918578:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695844.918579:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695844.918651:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695844.918655:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695844.918658:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787695845.238468:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695845.238481:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695845.238483:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695845.238489:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695845.238491:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695845.238494:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695845.238498:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695845.302490:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695845.302502:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695845.302506:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695845.302514:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695845.302519:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695845.302525:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695845.302528:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695845.302531:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695845.302534:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695845.750465:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695845.750465:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695845.750472:0:2406:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787695845.750474:0:2405:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787695845.750482:0:2403:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000400:2.0:1787695845.750495:0:2405:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695760/real 1787695760] req@ffff8b1446604000 x1874531139519104/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695845 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:3.0:1787695845.750495:0:2406:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695760/real 1787695760] req@ffff8b1446607100 x1874531139519232/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695845 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695845.750498:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695845.750505:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695845.750519:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695845.750529:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1787695845.750542:0:2406:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787695845.750546:0:2406:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1787695845.750549:0:2406:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14456d1e70. 00000400:00000200:3.0:1787695845.750555:0:2406:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b145078a990 00000400:00000010:3.0:1787695845.750560:0:2406:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b145078a990. 00000100:00000001:0.0:1787695845.750563:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695845.750564:0:2406:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000001:2.0:1787695845.750565:0:2405:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695845.750573:0:2405:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787695845.750575:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1787695845.750576:0:2406:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446607100 x1874531139519232/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695845 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1787695845.750578:0:2405:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bc08. 00000400:00000200:2.0:1787695845.750588:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8bb0 00000400:00000010:2.0:1787695845.750593:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8bb0. 00000100:00000200:3.0:1787695845.750597:0:2406:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446607100 x1874531139519232/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695845 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695845.750599:0:2405:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695845.750605:0:2405:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446604000 x1874531139519104/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695845 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695845.750620:0:2405:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446604000 x1874531139519104/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695845 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695845.750621:0:2406:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787695845.750624:0:2406:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695845.750629:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695845.750631:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695845.750637:0:2405:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:3.0:1787695845.750638:0:2406:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446607100 x1874531139519232/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695845 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695845.750642:0:2405:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695845.750648:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695845.750648:0:2406:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00000001:2.0:1787695845.750652:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695845.750654:0:2406:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b144515c000 already not connected (conn 23, was 11): DISCONN 00000100:00000001:3.0:1787695845.750657:0:2406:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:3.0:1787695845.750658:0:2406:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695845.750666:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695845.750669:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695845.750673:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1787695845.750676:0:2405:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446604000 x1874531139519104/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695845 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695845.750676:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695845.750682:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695845.750686:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695845.750688:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695845.750694:0:2405:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00000040:3.0:1787695845.750699:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446607100 x1874531139519232/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695845 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695845.750700:0:2405:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b1458081800 already not connected (conn 27, was 15): DISCONN 00000100:00000001:3.0:1787695845.750710:0:2406:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:3.0:1787695845.750712:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695845.750713:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695845.750715:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695845.750717:0:2405:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000040:3.0:1787695845.750719:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446607100 x1874531139519232/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695845 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695845.750721:0:2405:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695845.750731:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695845.750733:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695845.750736:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695845.750738:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695845.750741:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695845.750745:0:2405:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695845.750746:0:2405:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695845.750753:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446604000 x1874531139519104/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695845 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695845.750762:0:2405:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:2.0:1787695845.750765:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695845.750767:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695845.750769:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695845.750773:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446604000 x1874531139519104/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695845 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695845.750781:0:2405:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446604000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2405:1874531139519104:192.168.201.140@tcp:400:kworker.0 00000100:00000001:2.0:1787695845.750787:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695845.750790:0:2405:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695845.750793:0:2405:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446604000 x1874531139519104/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695845 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695845.750808:0:2405:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00100000:3.0:1787695845.750809:0:2406:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446607100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:608c526c-eb6e-4a54-80a5-8969828241eb:2406:1874531139519232:192.168.201.140@tcp:400:kworker.0 02000000:00000001:2.0:1787695845.750811:0:2405:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1787695845.750815:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695845.750816:0:2405:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1448928c00. 00000100:00000001:3.0:1787695845.750818:0:2406:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1787695845.750822:0:2405:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:3.0:1787695845.750823:0:2406:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446607100 x1874531139519232/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695845 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787695845.750826:0:2405:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695845.750833:0:2405:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=12 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:2.0:1787695845.750835:0:2405:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695845.750839:0:2405:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1458095400. 02000000:00000001:2.0:1787695845.750845:0:2405:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1787695845.750845:0:2406:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695845.750848:0:2405:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:3.0:1787695845.750849:0:2406:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000010:2.0:1787695845.750851:0:2405:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446604000. 02000000:00000010:3.0:1787695845.750852:0:2406:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b14474e2200. 00000100:00000001:2.0:1787695845.750856:0:2405:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 02000000:00000001:3.0:1787695845.750858:0:2406:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787695845.750859:0:2405:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787695845.750861:0:2406:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1787695845.750863:0:2406:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=13 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000001:2.0:1787695845.750864:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695845.750866:0:2406:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:3.0:1787695845.750898:0:2406:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1458095b00. 02000000:00000001:3.0:1787695845.750901:0:2406:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787695845.750902:0:2406:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1787695845.750904:0:2406:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446607100. 00000100:00000001:3.0:1787695845.750906:0:2406:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787695845.750907:0:2406:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695845.750909:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695845.942474:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695845.942482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695845.942488:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695845.942489:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695845.942492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695845.942493:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695845.942496:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695845.942497:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695845.942500:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695845.942501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695845.942504:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695845.942505:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695845.942613:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695845.942619:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695845.942623:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787695846.262792:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695846.262798:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:1.0:1787695846.262808:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787695846.262811:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695846.262812:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787695846.262814:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787695846.262818:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695846.262819:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695846.262823:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695846.262827:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787695846.262828:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.262832:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695846.262837:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3733 00000400:00000200:1.0:1787695846.262847:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695846.262853:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695846.262857:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695846.262873:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695846.262877:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695846.262880:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695846.262893:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695846.262898:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695846.262901:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:1.0:1787695846.262904:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695846.262908:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:1.0:1787695846.262911:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:1.0:1787695846.262914:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:1.0:1787695846.262918:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695846.262920:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695846.262921:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695846.262923:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695846.262926:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:1.0:1787695846.262929:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695846.262930:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695846.262933:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695846.262936:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695846.262939:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695846.262941:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695846.262943:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695846.262949:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665180. 00000020:00000040:1.0:1787695846.262952:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:1.0:1787695846.262956:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695846.262957:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695846.262960:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695846.262963:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695846.262968:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695846.262976:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446546800. 00000100:00000001:1.0:1787695846.262983:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695846.262993:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446665180 x1874531139532160/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695846.263007:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695846.263011:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695846.263015:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f923372e] to pc [ptlrpcd_rcv+-1] req@ffff8b1446665180 x1874531139532160/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695846.263032:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695846.263035:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695846.263037:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787695846.263038:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695846.263040:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695846.263043:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695846.263045:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3733 00000400:00000200:1.0:1787695846.263049:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695846.263052:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695846.263054:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695846.263057:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695846.263058:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695846.263059:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695846.263062:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1787695846.263063:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695846.263066:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787695846.263067:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695846.263070:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695846.263071:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:1.0:1787695846.263085:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695846.263087:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:1.0:1787695846.263089:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:0.0:1787695846.263090:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665180 x1874531139532160/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787695846.263091:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:1.0:1787695846.263099:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695846.263103:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695846.263107:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1787695846.263108:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695846.263110:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695846.263111:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:0.0:1787695846.263116:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695846.263120:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1787695846.263121:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 02000000:00000001:0.0:1787695846.263124:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695846.263127:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695846.263131:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695846.263133:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139532160:192.168.201.140@tcp:250:kworker.0 02000000:00000001:1.0:1787695846.263133:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695846.263139:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695846.263140:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695846.263142:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695846.263145:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787695846.263148:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:1.0:1787695846.263150:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787695846.263153:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1787695846.263154:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695846.263159:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695846.263160:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:1.0:1787695846.263169:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446666680. 02000000:00000010:0.0:1787695846.263170:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695846.263174:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1787695846.263177:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:1.0:1787695846.263180:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695846.263181:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695846.263186:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695846.263186:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695846.263189:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695846.263191:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695846.263195:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617be18. 02000000:00000010:1.0:1787695846.263196:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545800. 00000100:00000001:1.0:1787695846.263201:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695846.263204:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00100000:1.0:1787695846.263206:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446666680 x1874531139532288/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:0.0:1787695846.263213:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139532160, portal 25 00000100:00000001:1.0:1787695846.263214:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695846.263219:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695846.263219:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.263223:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929558400 : -128555779993216 : ffff8b1446665180) 00000100:00000040:1.0:1787695846.263228:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bd4d567] to pc [ptlrpcd_rcv+-1] req@ffff8b1446666680 x1874531139532288/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695846.263231:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665180 x1874531139532160/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695846.263238:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695846.263244:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695846.263251:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1787695846.263252:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000040:0.0:1787695846.263258:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787695846.263258:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:0.0:1787695846.263261:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00080000:1.0:1787695846.263263:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:0.0:1787695846.263266:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139532160, offset 0 00000100:00000001:1.0:1787695846.263269:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:0.0:1787695846.263273:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695846.263283:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:1.0:1787695846.263283:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3733 00000400:00000200:0.0:1787695846.263293:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695846.263298:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695846.263303:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695846.263307:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695846.263310:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695846.263313:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16967:16967:256:4294967295] 192.168.201.140@tcp LPNI seq info [16966:16966:8:4294967295] 00000400:00000200:0.0:1787695846.263317:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695846.263324:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695846.263330:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000400:00000200:1.0:1787695846.263335:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695846.263339:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000800:00000200:0.0:1787695846.263341:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695846.263348:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfcd00. 00000100:00000040:1.0:1787695846.263349:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695846.263354:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.0:1787695846.263358:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:1.0:1787695846.263360:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000200:0.0:1787695846.263386:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 23 secs later) 00000100:00000001:1.0:1787695846.263403:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000200:0.1:1787695846.263404:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000020:00000001:1.0:1787695846.263408:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695846.263412:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000800:00000200:0.1:1787695846.263414:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 23 secs later) 00000020:00000001:1.0:1787695846.263416:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695846.263421:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:1.0:1787695846.263425:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000800:00000100:0.0:1787695846.263426:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000020:00000001:1.0:1787695846.263429:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000800:00000010:0.0:1787695846.263432:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfcd00. 00000100:00000040:1.0:1787695846.263435:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695846.263438:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695846.263440:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695846.263442:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:0.0:1787695846.263443:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:1.0:1787695846.263444:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:1.0:1787695846.263450:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695846.263453:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695846.263456:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 02000000:00000001:1.0:1787695846.263457:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695846.263461:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1787695846.263465:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 02000000:00000001:1.0:1787695846.263465:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695846.263467:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:0.0:1787695846.263469:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000008:00100000:1.0:1787695846.263470:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787695846.263473:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.263474:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787695846.263475:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695846.263481:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446665180 x1874531139532160/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1787695846.263482:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446664000. 00000020:00000040:1.0:1787695846.263488:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=13 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:1.0:1787695846.263496:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787695846.263497:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695846.263500:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665180 x1874531139532160/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695871 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695846.263507:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695846.263511:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695846.263512:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695846.263515:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.263524:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695846.263524:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545400. 00000100:00000001:0.0:1787695846.263529:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695846.263532:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.263538:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695846.263541:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446664000 x1874531139532416/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1787695846.263545:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695846.263550:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298417977 00000100:00000001:1.0:1787695846.263559:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695846.263560:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298412856 00000100:00000001:1.0:1787695846.263565:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.263567:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298417977 00000100:00000001:0.0:1787695846.263572:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298412856 00000100:00000040:1.0:1787695846.263573:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e60c2a2d] to pc [ptlrpcd_rcv+-1] req@ffff8b1446664000 x1874531139532416/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.263576:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695846.263581:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.263585:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1787695846.263591:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695846.263593:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695846/real 1787695846] req@ffff8b1446665180 x1874531139532160/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695846.263594:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695846.263603:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787695846.263606:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695846.263610:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:0.0:1787695846.263612:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1787695846.263613:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000001:0.0:1787695846.263615:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:1.0:1787695846.263618:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3733 00000400:00000010:0.0:1787695846.263619:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617be18. 00000400:00000200:0.0:1787695846.263625:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000400:00000200:1.0:1787695846.263626:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695846.263627:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000001:0.0:1787695846.263631:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:1.0:1787695846.263631:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695846.263635:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000200:0.0:1787695846.263638:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665180 x1874531139532160/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695846.263641:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695846.263647:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695846.263651:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695846.263655:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000200:0.0:1787695846.263656:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665180 x1874531139532160/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787695846.263658:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787695846.263663:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695846.263664:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.263666:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:1.0:1787695846.263669:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000100:00000001:0.0:1787695846.263670:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787695846.263673:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:0.0:1787695846.263676:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446665180 x1874531139532160/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1787695846.263677:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:1.0:1787695846.263682:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:1.0:1787695846.263691:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:1.0:1787695846.263699:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695846.263703:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695846.263703:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695846.263707:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695846.263710:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000040:0.0:1787695846.263712:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665180 x1874531139532160/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1787695846.263713:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:1.0:1787695846.263719:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695846.263725:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695846.263729:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787695846.263731:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1787695846.263733:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695846.263734:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.263739:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1787695846.263739:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.263742:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695846.263743:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787695846.263745:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000008:00100000:1.0:1787695846.263747:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787695846.263748:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:1.0:1787695846.263752:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695846.263753:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1787695846.263756:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695846.263760:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695846.263763:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000010:1.0:1787695846.263763:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446664a80. 00000020:00000040:1.0:1787695846.263769:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000100:00000040:0.0:1787695846.263770:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665180 x1874531139532160/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695846.263775:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695846.263779:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695846.263781:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139532160:192.168.201.140@tcp:250:kworker.0 00000100:00000001:1.0:1787695846.263784:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787695846.263786:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787695846.263788:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787695846.263789:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695846.263793:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665180 x1874531139532160/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695846.263793:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695846.263801:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446547400. 00000100:00000001:1.0:1787695846.263807:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.263809:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695846.263812:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695846.263816:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450775c00. 00000100:00100000:1.0:1787695846.263817:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446664a80 x1874531139532544/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787695846.263819:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695846.263822:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695846.263824:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:0.0:1787695846.263829:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695846.263832:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446546800. 00000100:00000001:1.0:1787695846.263833:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695846.263836:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695846.263838:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:1.0:1787695846.263840:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ad35ba92] to pc [ptlrpcd_rcv+-1] req@ffff8b1446664a80 x1874531139532544/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787695846.263846:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695846.263851:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665180. 00000100:00000001:1.0:1787695846.263851:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.263857:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695846.263859:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.263866:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787695846.263867:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695846.263873:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1787695846.263873:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1787695846.263877:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695846.263878:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787695846.263883:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1787695846.263883:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695846.263887:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695846.263890:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787695846.263891:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1787695846.263897:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3733 00000100:00000040:0.0:1787695846.263900:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446666680 x1874531139532288/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695846.263907:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695846.263915:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:0.0:1787695846.263916:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:1.0:1787695846.263918:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787695846.263919:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695846.263924:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1787695846.263925:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:0.0:1787695846.263926:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695846.263928:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787695846.263930:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695846.263932:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00100000:0.0:1787695846.263936:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446666680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139532288:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695846.263937:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695846.263942:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787695846.263943:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:1.0:1787695846.263944:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 02000000:00000001:0.0:1787695846.263946:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:1.0:1787695846.263947:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1787695846.263948:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695846.263951:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:1.0:1787695846.263951:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:1.0:1787695846.263955:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 02000000:00000010:0.0:1787695846.263957:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450775c00. 00000020:00000001:1.0:1787695846.263958:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 02000000:00000001:0.0:1787695846.263961:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695846.263964:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695846.263967:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1787695846.263968:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:1.0:1787695846.263971:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000010:0.0:1787695846.263973:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617be18. 00000100:00000001:1.0:1787695846.263974:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695846.263979:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000400:00000010:0.0:1787695846.263980:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00000200:0.0:1787695846.263985:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139532288, portal 10 00000100:00080000:1.0:1787695846.263985:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695846.263989:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695846.263992:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929563776 : -128555779987840 : ffff8b1446666680) 00000100:00000001:1.0:1787695846.263993:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695846.263998:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:0.0:1787695846.264001:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446666680 x1874531139532288/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695846.264003:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695846.264010:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.264011:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695846.264016:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787695846.264017:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:0.0:1787695846.264022:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695846.264025:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139532288, offset 0 00000008:00100000:1.0:1787695846.264029:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000400:00000200:0.0:1787695846.264031:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000008:00000001:1.0:1787695846.264036:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695846.264037:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695846.264042:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695846.264043:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695846.264048:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000010:1.0:1787695846.264050:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665880. 00000400:00000200:0.0:1787695846.264051:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695846.264056:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000020:00000040:1.0:1787695846.264057:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 00000400:00000200:0.0:1787695846.264059:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 02000000:00000001:1.0:1787695846.264064:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:0.0:1787695846.264065:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16968:16968:256:4294967295] 192.168.201.140@tcp LPNI seq info [16967:16967:8:4294967295] 02000000:00000001:1.0:1787695846.264069:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695846.264085:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695846.264088:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695846.264091:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695846.264096:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695846.264097:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695846.264103:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 02000000:00000010:1.0:1787695846.264105:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446546800. 00000800:00000010:0.0:1787695846.264107:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfcd00. 00000100:00000001:1.0:1787695846.264112:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1787695846.264113:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695846.264118:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 23 secs later) 00000100:00100000:1.0:1787695846.264124:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446665880 x1874531139532672/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:0.1:1787695846.264125:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695846.264130:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 23 secs later) 00000800:00000100:0.0:1787695846.264142:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787695846.264145:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000010:0.0:1787695846.264146:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfcd00. 00000100:00000001:1.0:1787695846.264151:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695846.264153:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:1.0:1787695846.264160:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000716595a9] to pc [ptlrpcd_rcv+-1] req@ffff8b1446665880 x1874531139532672/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695846.264163:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695846.264170:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695846.264175:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:1.0:1787695846.264177:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.264180:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:1.0:1787695846.264183:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:0.0:1787695846.264186:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446666680 x1874531139532288/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695846.264189:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787695846.264195:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787695846.264199:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1787695846.264199:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:0.0:1787695846.264204:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446666680 x1874531139532288/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695871 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695846.264208:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787695846.264213:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695846.264215:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3733 00000100:00000001:0.0:1787695846.264216:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:1.0:1787695846.264222:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695846.264229:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695846.264230:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000040:0.0:1787695846.264236:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000100:00080000:1.0:1787695846.264237:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1787695846.264242:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:1.0:1787695846.264245:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695846.264246:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298407878 00000100:00000001:1.0:1787695846.264252:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695846.264253:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298417977 00000100:00000001:0.0:1787695846.264257:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298407878 00000100:00000040:1.0:1787695846.264259:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695846.264260:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298417977 00000100:00000001:0.0:1787695846.264262:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695846.264263:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.264264:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695846.264269:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695846/real 1787695846] req@ffff8b1446666680 x1874531139532288/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695846.264273:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000001:0.0:1787695846.264280:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:1.0:1787695846.264281:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000001:0.0:1787695846.264283:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695846.264285:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617be18. 00000020:00000040:1.0:1787695846.264285:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000400:00000200:0.0:1787695846.264291:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000020:00000001:1.0:1787695846.264291:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000010:0.0:1787695846.264295:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000001:0.0:1787695846.264297:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:1.0:1787695846.264297:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000100:00000200:0.0:1787695846.264302:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446666680 x1874531139532288/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787695846.264303:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000020:00000001:1.0:1787695846.264310:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000200:0.0:1787695846.264317:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446666680 x1874531139532288/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695846.264321:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695846.264327:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695846.264328:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695846.264330:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695846.264332:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695846.264333:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695846.264335:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695846.264337:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00100000:0.0:1787695846.264340:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446666680 x1874531139532288/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1787695846.264345:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00000001:0.0:1787695846.264350:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787695846.264355:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000040:0.0:1787695846.264356:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446666680 x1874531139532288/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695846.264360:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.264385:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1787695846.264390:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787695846.264391:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.264395:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1787695846.264396:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695846.264397:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.264404:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:1.0:1787695846.264404:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.264408:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695846.264410:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00080000:0.0:1787695846.264415:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1787695846.264415:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787695846.264422:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695846.264424:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695846.264425:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.264430:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000010:1.0:1787695846.264433:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446667800. 00000100:00000040:0.0:1787695846.264440:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446666680 x1874531139532288/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1787695846.264440:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:1.0:1787695846.264446:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695846.264449:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695846.264454:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:0.0:1787695846.264458:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446666680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139532288:192.168.201.140@tcp:38:kworker.0 02000000:00000001:1.0:1787695846.264458:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695846.264463:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.264464:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695846.264469:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446664000 x1874531139532416/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1787695846.264473:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545c00. 00000100:00000001:0.0:1787695846.264475:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695846.264478:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695846.264480:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1787695846.264481:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695846.264482:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695846.264486:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695846.264492:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446664000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139532416:192.168.201.140@tcp:38:kworker.0 00000100:00100000:1.0:1787695846.264493:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446667800 x1874531139532800/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695846.264496:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695846.264499:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695846.264502:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695846.264505:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695846.264509:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695846.264512:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695846.264522:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1787695846.264523:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695846.264528:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617be18. 00000100:00000001:1.0:1787695846.264529:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695846.264531:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00000200:0.0:1787695846.264536:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139532416, portal 10 00000100:00000001:0.0:1787695846.264539:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:1.0:1787695846.264539:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009df211f4] to pc [ptlrpcd_rcv+-1] req@ffff8b1446667800 x1874531139532800/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.264540:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929553920 : -128555779997696 : ffff8b1446664000) 00000100:00000040:0.0:1787695846.264550:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446664000 x1874531139532416/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.264561:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787695846.264561:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695846.264566:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00080000:1.0:1787695846.264568:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:0.0:1787695846.264569:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695846.264573:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139532416, offset 0 00000100:00080000:1.0:1787695846.264576:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:0.0:1787695846.264580:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787695846.264581:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:0.0:1787695846.264584:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:1.0:1787695846.264585:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:0.0:1787695846.264589:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:1.0:1787695846.264592:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:0.0:1787695846.264594:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:1.0:1787695846.264598:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3733 00000400:00000200:0.0:1787695846.264600:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695846.264604:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695846.264605:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695846.264607:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16969:16969:256:4294967295] 192.168.201.140@tcp LPNI seq info [16968:16968:8:4294967295] 00000400:00000200:0.0:1787695846.264610:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695846.264615:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695846.264620:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000400:00000200:1.0:1787695846.264623:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695846.264627:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000800:00000200:0.0:1787695846.264628:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695846.264633:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfcd00. 00000100:00000040:1.0:1787695846.264634:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695846.264638:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:0.0:1787695846.264641:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:1.0:1787695846.264643:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000200:0.0:1787695846.264645:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 23 secs later) 00000100:00000001:1.0:1787695846.264646:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695846.264652:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695846.264655:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000800:00000200:0.1:1787695846.264656:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000020:00000001:1.0:1787695846.264658:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000200:0.1:1787695846.264660:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 23 secs later) 00000020:00000040:1.0:1787695846.264661:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:1.0:1787695846.264667:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000800:00000100:0.0:1787695846.264670:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000020:00000001:1.0:1787695846.264670:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000800:00000010:0.0:1787695846.264673:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfcd00. 00000100:00000040:1.0:1787695846.264677:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695846.264679:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1787695846.264680:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695846.264685:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695846.264686:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1787695846.264688:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:0.0:1787695846.264691:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000020:00000040:1.0:1787695846.264697:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000400:00000010:0.0:1787695846.264699:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00080000:1.0:1787695846.264703:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695846.264705:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787695846.264708:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695846.264712:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446664000 x1874531139532416/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695846.264713:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695846.264717:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695846.264722:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695846.264727:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787695846.264728:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000008:00100000:1.0:1787695846.264730:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000040:0.0:1787695846.264733:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446664000 x1874531139532416/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695871 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1787695846.264734:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695846.264738:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695846.264743:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665180. 00000100:00000001:0.0:1787695846.264745:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1787695846.264747:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=14 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000001:0.0:1787695846.264748:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695846.264753:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695846.264753:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695846.264755:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695846.264759:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000100:00000001:1.0:1787695846.264759:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695846.264762:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695846.264765:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695846.264766:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695846.264771:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298406727 00000100:00000001:0.0:1787695846.264774:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298417977 02000000:00000010:1.0:1787695846.264775:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446547c00. 00000100:00000001:0.0:1787695846.264776:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298406727 00000100:00000001:0.0:1787695846.264781:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298417977 00000100:00000001:0.0:1787695846.264783:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695846.264783:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.264788:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.264791:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695846.264795:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446665180 x1874531139532928/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787695846.264796:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695846/real 1787695846] req@ffff8b1446664000 x1874531139532416/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695846.264825:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695846.264827:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787695846.264828:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000010:0.0:1787695846.264829:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617be18. 00000400:00000200:0.0:1787695846.264833:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000100:00000001:1.0:1787695846.264833:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695846.264838:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000040:1.0:1787695846.264841:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f923372e] to pc [ptlrpcd_rcv+-1] req@ffff8b1446665180 x1874531139532928/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.264842:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695846.264848:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446664000 x1874531139532416/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695846.264856:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695846.264861:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1787695846.264869:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000200:0.0:1787695846.264871:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446664000 x1874531139532416/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695846.264873:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1787695846.264881:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1787695846.264888:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695846.264891:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.264895:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:1.0:1787695846.264897:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3744) 00000100:00000001:0.0:1787695846.264898:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695846.264907:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446664000 x1874531139532416/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.264912:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695846.264916:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446664000 x1874531139532416/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.264921:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695846.264923:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.264924:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695846.264924:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.264926:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695846.264927:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695846.264929:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695846.264931:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695846.264933:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695846.264936:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446664000 x1874531139532416/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695846.264940:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446664000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139532416:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695846.264943:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695846.264946:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446664a80 x1874531139532544/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.264950:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695846.264951:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695846.264957:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695846.264958:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695846.264959:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695846.264962:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446664a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139532544:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695846.264964:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695846.264966:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695846.264967:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695846.264968:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695846.264971:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450777000. 02000000:00000001:0.0:1787695846.264972:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695846.264976:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695846.264978:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617be18. 00000400:00000010:0.0:1787695846.264981:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00000200:0.0:1787695846.264983:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139532544, portal 4 00000100:00000001:0.0:1787695846.264984:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695846.264986:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929556608 : -128555779995008 : ffff8b1446664a80) 00000100:00000040:0.0:1787695846.264990:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446664a80 x1874531139532544/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.264995:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695846.264997:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695846.264998:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695846.265000:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139532544, offset 0 00000400:00000200:0.0:1787695846.265002:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695846.265005:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695846.265007:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695846.265010:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695846.265011:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695846.265013:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695846.265015:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695846.265017:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16970:16970:256:4294967295] 192.168.201.140@tcp LPNI seq info [16969:16969:8:4294967295] 00000400:00000200:0.0:1787695846.265020:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695846.265025:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695846.265032:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695846.265034:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfcd00. 00000800:00000200:0.0:1787695846.265037:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695846.265038:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 23 secs later) 00000800:00000200:0.1:1787695846.265042:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695846.265043:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 23 secs later) 00000800:00000100:0.0:1787695846.265046:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695846.265048:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfcd00. 00000400:00000200:0.0:1787695846.265050:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695846.265052:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695846.265055:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695846.265056:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695846.265058:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695846.265060:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446664a80 x1874531139532544/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265066:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695846.265068:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446664a80 x1874531139532544/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695871 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265085:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.265087:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695846.265090:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695846.265092:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:0.0:1787695846.265095:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695846.265097:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298406757 00000100:00000001:0.0:1787695846.265099:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298417977 00000100:00000001:0.0:1787695846.265100:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298406757 00000100:00000001:0.0:1787695846.265102:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298417977 00000100:00000001:0.0:1787695846.265104:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695846.265105:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.265107:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695846.265109:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695846/real 1787695846] req@ffff8b1446664a80 x1874531139532544/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695846.265121:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695846.265122:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695846.265123:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617be18. 00000400:00000200:0.0:1787695846.265125:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000400:00000010:0.0:1787695846.265127:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000001:0.0:1787695846.265129:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695846.265133:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446664a80 x1874531139532544/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695846.265142:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446664a80 x1874531139532544/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265149:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695846.265151:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.265153:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695846.265155:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695846.265158:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446664a80 x1874531139532544/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265166:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695846.265171:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446664a80 x1874531139532544/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265179:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695846.265182:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.265183:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695846.265185:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.265187:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695846.265189:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695846.265192:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695846.265195:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695846.265198:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695846.265209:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446664a80 x1874531139532544/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695846.265219:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446664a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139532544:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695846.265223:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.265225:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695846.265229:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446666680 x1874531139532288/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265235:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695846.265236:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695846.265238:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695846.265240:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695846.265241:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695846.265242:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695846.265244:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695846.265245:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545800. 02000000:00000001:0.0:1787695846.265249:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695846.265250:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695846.265251:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446666680. 00000100:00000001:0.0:1787695846.265253:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695846.265254:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.265255:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695846.265258:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446664000 x1874531139532416/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265262:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695846.265263:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695846.265264:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695846.265266:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695846.265267:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695846.265268:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=12 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695846.265269:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695846.265270:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545400. 02000000:00000001:0.0:1787695846.265271:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695846.265275:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695846.265276:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446664000. 00000100:00000001:0.0:1787695846.265278:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695846.265279:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.265280:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695846.265283:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446664a80 x1874531139532544/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265287:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695846.265288:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695846.265288:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450777000. 02000000:00000001:0.0:1787695846.265290:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695846.265291:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695846.265291:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:0.0:1787695846.265292:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695846.265293:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446547400. 02000000:00000001:0.0:1787695846.265296:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695846.265296:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695846.265297:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446664a80. 00000100:00000001:0.0:1787695846.265298:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695846.265299:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.265300:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.265303:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695846.265304:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.265305:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695846.265306:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695846.265307:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695846.265309:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665880 x1874531139532672/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265314:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695846.265315:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695846.265317:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695846.265318:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695846.265320:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695846.265321:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139532672:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695846.265323:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695846.265328:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695846.265329:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695846.265330:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695846.265332:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450777000. 02000000:00000001:0.0:1787695846.265334:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695846.265336:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695846.265338:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617be18. 00000400:00000010:0.0:1787695846.265340:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00000200:0.0:1787695846.265343:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139532672, portal 4 00000100:00000001:0.0:1787695846.265344:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695846.265345:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929560192 : -128555779991424 : ffff8b1446665880) 00000100:00000040:0.0:1787695846.265350:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665880 x1874531139532672/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265355:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695846.265357:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695846.265359:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695846.265360:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139532672, offset 0 00000400:00000200:0.0:1787695846.265378:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695846.265381:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695846.265385:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695846.265388:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695846.265391:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695846.265394:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695846.265397:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695846.265402:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16971:16971:256:4294967295] 192.168.201.140@tcp LPNI seq info [16970:16970:8:4294967295] 00000400:00000200:0.0:1787695846.265407:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695846.265419:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695846.265425:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695846.265429:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfcd00. 00000800:00000200:0.0:1787695846.265434:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695846.265437:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 23 secs later) 00000800:00000200:0.1:1787695846.265442:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695846.265445:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 23 secs later) 00000800:00000100:0.0:1787695846.265448:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695846.265454:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfcd00. 00000400:00000200:0.0:1787695846.265456:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695846.265459:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695846.265462:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695846.265463:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695846.265465:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695846.265468:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446665880 x1874531139532672/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265474:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695846.265476:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665880 x1874531139532672/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695871 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265480:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.265481:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695846.265484:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695846.265486:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:0.0:1787695846.265488:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695846.265490:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298406430 00000100:00000001:0.0:1787695846.265492:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298417977 00000100:00000001:0.0:1787695846.265493:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298406430 00000100:00000001:0.0:1787695846.265495:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298417977 00000100:00000001:0.0:1787695846.265496:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695846.265498:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.265499:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695846.265501:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695846/real 1787695846] req@ffff8b1446665880 x1874531139532672/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695846.265506:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695846.265507:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695846.265509:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617be18. 00000400:00000200:0.0:1787695846.265511:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000400:00000010:0.0:1787695846.265512:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000001:0.0:1787695846.265517:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695846.265519:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665880 x1874531139532672/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695846.265525:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665880 x1874531139532672/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265529:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695846.265530:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.265531:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695846.265532:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695846.265534:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446665880 x1874531139532672/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265538:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695846.265540:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665880 x1874531139532672/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265545:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695846.265547:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.265548:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695846.265549:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.265550:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695846.265551:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695846.265552:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695846.265554:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695846.265555:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695846.265558:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665880 x1874531139532672/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695846.265562:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139532672:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695846.265565:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695846.265567:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446667800 x1874531139532800/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265574:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695846.265576:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695846.265577:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695846.265579:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695846.265580:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695846.265582:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446667800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139532800:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695846.265584:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695846.265586:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695846.265587:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695846.265588:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695846.265590:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695846.265591:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695846.265594:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695846.265595:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617be18. 00000400:00000010:0.0:1787695846.265598:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00000200:0.0:1787695846.265600:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139532800, portal 10 00000100:00000001:0.0:1787695846.265601:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695846.265603:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929568256 : -128555779983360 : ffff8b1446667800) 00000100:00000040:0.0:1787695846.265608:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446667800 x1874531139532800/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265612:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695846.265614:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695846.265615:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695846.265617:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139532800, offset 0 00000400:00000200:0.0:1787695846.265619:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695846.265622:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695846.265624:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695846.265626:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695846.265627:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695846.265629:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695846.265631:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695846.265638:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16972:16972:256:4294967295] 192.168.201.140@tcp LPNI seq info [16971:16971:8:4294967295] 00000400:00000200:0.0:1787695846.265641:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695846.265646:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695846.265649:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695846.265651:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfcd00. 00000800:00000200:0.0:1787695846.265654:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695846.265656:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 23 secs later) 00000800:00000200:0.1:1787695846.265659:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695846.265661:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 23 secs later) 00000800:00000100:0.0:1787695846.265663:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695846.265665:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfcd00. 00000400:00000200:0.0:1787695846.265667:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695846.265669:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695846.265671:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695846.265673:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695846.265674:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695846.265676:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446667800 x1874531139532800/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265682:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695846.265684:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446667800 x1874531139532800/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695871 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265688:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.265689:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695846.265691:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695846.265693:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:0.0:1787695846.265695:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695846.265696:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298406616 00000100:00000001:0.0:1787695846.265698:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298417977 00000100:00000001:0.0:1787695846.265699:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298406616 00000100:00000001:0.0:1787695846.265701:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298417977 00000100:00000001:0.0:1787695846.265703:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695846.265707:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.265708:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695846.265711:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695846/real 1787695846] req@ffff8b1446667800 x1874531139532800/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695846.265717:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695846.265718:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695846.265719:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617be18. 00000400:00000200:0.0:1787695846.265720:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000400:00000010:0.0:1787695846.265721:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000001:0.0:1787695846.265722:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695846.265724:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446667800 x1874531139532800/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695846.265730:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446667800 x1874531139532800/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265734:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695846.265735:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.265736:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695846.265737:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695846.265739:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446667800 x1874531139532800/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265743:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695846.265745:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446667800 x1874531139532800/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265749:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695846.265750:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.265751:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695846.265752:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.265753:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695846.265754:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695846.265756:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695846.265761:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695846.265762:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695846.265765:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446667800 x1874531139532800/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695846.265770:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446667800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139532800:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695846.265773:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695846.265775:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665180 x1874531139532928/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265778:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695846.265779:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695846.265781:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695846.265782:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695846.265783:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695846.265785:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139532928:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695846.265787:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695846.265789:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695846.265790:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695846.265791:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695846.265793:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695846.265794:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695846.265796:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695846.265797:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617be18. 00000400:00000010:0.0:1787695846.265799:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00000200:0.0:1787695846.265801:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139532928, portal 10 00000100:00000001:0.0:1787695846.265803:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695846.265804:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929558400 : -128555779993216 : ffff8b1446665180) 00000100:00000040:0.0:1787695846.265808:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665180 x1874531139532928/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265813:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695846.265817:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695846.265818:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96880. 00000100:00000200:0.0:1787695846.265820:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139532928, offset 0 00000400:00000200:0.0:1787695846.265823:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695846.265826:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695846.265828:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695846.265830:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695846.265832:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695846.265833:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695846.265835:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695846.265837:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16973:16973:256:4294967295] 192.168.201.140@tcp LPNI seq info [16972:16972:8:4294967295] 00000400:00000200:0.0:1787695846.265840:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695846.265844:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695846.265847:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695846.265850:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfcd00. 00000800:00000200:0.0:1787695846.265853:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695846.265856:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 23 secs later) 00000800:00000200:0.1:1787695846.265861:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695846.265863:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 23 secs later) 00000800:00000100:0.0:1787695846.265867:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695846.265869:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfcd00. 00000400:00000200:0.0:1787695846.265872:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695846.265875:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695846.265878:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96880 00000400:00000010:0.0:1787695846.265880:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96880. 00000100:00000001:0.0:1787695846.265883:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695846.265887:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446665180 x1874531139532928/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695871 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265896:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695846.265900:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665180 x1874531139532928/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695871 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265906:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.265908:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695846.265915:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695846.265918:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:0.0:1787695846.265920:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695846.265922:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298406594 00000100:00000001:0.0:1787695846.265924:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298417977 00000100:00000001:0.0:1787695846.265925:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298406594 00000100:00000001:0.0:1787695846.265927:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298417977 00000100:00000001:0.0:1787695846.265929:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695846.265930:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.265931:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695846.265934:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695846/real 1787695846] req@ffff8b1446665180 x1874531139532928/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695846.265939:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695846.265940:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695846.265941:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617be18. 00000400:00000200:0.0:1787695846.265943:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000400:00000010:0.0:1787695846.265944:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000001:0.0:1787695846.265945:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695846.265948:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665180 x1874531139532928/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695846.265953:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665180 x1874531139532928/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265957:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695846.265958:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.265959:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695846.265960:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695846.265962:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446665180 x1874531139532928/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265966:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695846.265968:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665180 x1874531139532928/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.265977:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695846.265978:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.265979:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695846.265980:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.265981:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695846.265982:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695846.265984:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695846.265986:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695846.265987:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695846.265990:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665180 x1874531139532928/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695846.265995:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139532928:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695846.265998:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.265999:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695846.266001:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665880 x1874531139532672/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.266005:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695846.266006:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695846.266007:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450777000. 02000000:00000001:0.0:1787695846.266009:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695846.266010:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695846.266010:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:0.0:1787695846.266012:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695846.266013:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446546800. 02000000:00000001:0.0:1787695846.266015:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695846.266016:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695846.266018:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665880. 00000100:00000001:0.0:1787695846.266019:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695846.266020:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.266025:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695846.266027:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446667800 x1874531139532800/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.266032:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695846.266033:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695846.266034:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695846.266035:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695846.266036:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695846.266037:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695846.266038:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695846.266039:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545c00. 02000000:00000001:0.0:1787695846.266040:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695846.266041:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695846.266042:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446667800. 00000100:00000001:0.0:1787695846.266044:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695846.266045:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.266046:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695846.266047:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665180 x1874531139532928/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695871 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695846.266051:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695846.266052:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695846.266053:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450775c00. 02000000:00000001:0.0:1787695846.266054:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695846.266055:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695846.266056:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=13 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695846.266057:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695846.266058:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446547c00. 02000000:00000001:0.0:1787695846.266060:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695846.266060:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695846.266061:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665180. 00000100:00000001:0.0:1787695846.266062:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695846.266063:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.266064:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.266067:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695846.266067:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.266069:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695846.266086:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695846.326435:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695846.326441:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695846.326443:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695846.326447:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695846.326450:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695846.326453:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695846.326455:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695846.326458:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695846.326460:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.774441:0:2403:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695846.774441:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695846.774452:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695846.774454:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695846.774454:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695846.774464:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695846.774464:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695846.774469:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695846.774471:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695846.774472:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695846.774476:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695846.774477:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695846.774477:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695846.774488:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695846.774489:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695846.774489:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695846.774493:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695846.774497:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695846.774500:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695846.774500:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695846.774508:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695846.966473:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695846.966481:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695846.966487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695846.966489:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695846.966494:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695846.966497:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695846.966499:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695846.966504:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695846.966506:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695846.966507:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695846.966510:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695846.966512:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695846.966513:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695846.966515:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695846.966518:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695846.966519:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695846.966520:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695846.966521:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695846.966522:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695846.966522:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695846.966523:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695846.966524:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695846.966525:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695846.966526:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695846.966529:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695846.966530:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695846.966532:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695846.966533:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695846.966534:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695846.966535:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695846.966536:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695846.966537:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695846.966538:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695846.966538:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695846.966539:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695846.966541:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695846.966602:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695846.966606:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695846.966609:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787695847.287410:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695847.287420:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695847.287422:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695847.287426:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695847.287428:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695847.287431:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695847.287434:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695847.350505:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695847.350514:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695847.350517:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695847.350524:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695847.350529:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695847.350533:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695847.350559:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695847.350563:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695847.350567:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695847.798455:0:2403:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695847.798455:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695847.798455:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695847.798468:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695847.798468:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695847.798469:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695847.798472:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695847.798473:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695847.798474:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695847.798482:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695847.798482:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695847.798485:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695847.798486:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695847.798487:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695847.798488:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695847.798494:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695847.798495:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695847.798495:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695847.798498:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695847.798501:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695847.798503:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695847.990486:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695847.990510:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695847.990517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695847.990518:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695847.990522:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695847.990524:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695847.990527:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695847.990528:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695847.990532:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695847.990533:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695847.990537:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695847.990538:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695847.990593:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695847.990596:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695847.990598:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787695848.310686:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695848.310698:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695848.310699:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695848.310703:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695848.310705:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695848.310709:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695848.310711:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695848.374419:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695848.374425:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695848.374427:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695848.374431:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695848.374434:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695848.374436:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695848.374438:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695848.374439:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695848.374442:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695848.822431:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695848.822432:0:2403:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695848.822432:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695848.822444:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695848.822444:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695848.822444:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695848.822449:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695848.822449:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695848.822450:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695848.822459:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695848.822459:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695848.822459:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695848.822463:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695848.822464:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695848.822464:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695848.822471:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695848.822471:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695848.822472:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695848.822475:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695848.822481:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695848.822481:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695849.014504:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695849.014511:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695849.014517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695849.014518:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695849.014522:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695849.014523:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695849.014526:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695849.014527:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695849.014530:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695849.014531:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695849.014533:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695849.014534:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695849.014609:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695849.014615:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695849.014619:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787695849.334533:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695849.334540:0:2404:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000400:1.0:1787695849.334564:0:2404:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695764/real 1787695764] req@ffff8b1446667480 x1874531139519488/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695849 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695849.334583:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695849.334585:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695849.334589:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702c08. 00000400:00000200:1.0:1787695849.334608:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:1.0:1787695849.334611:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000100:00000001:1.0:1787695849.334616:0:2404:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695849.334621:0:2404:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446667480 x1874531139519488/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695849 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695849.334632:0:2404:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446667480 x1874531139519488/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695849 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695849.334641:0:2404:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695849.334643:0:2404:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695849.334646:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695849.334648:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695849.334651:0:2404:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446667480 x1874531139519488/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695849 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695849.334657:0:2404:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:1.0:1787695849.334659:0:2404:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b144515c000 already not connected (conn 24, was 11): DISCONN 00000100:00000001:1.0:1787695849.334661:0:2404:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:1.0:1787695849.334662:0:2404:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695849.334669:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695849.334670:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=7 : 7 : 7) 00000100:00000001:1.0:1787695849.334672:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695849.334675:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695849.334677:0:2404:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695849.334679:0:2404:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695849.334680:0:2404:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695849.334683:0:2404:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446667480 x1874531139519488/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695849 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695849.334689:0:2404:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:1.0:1787695849.334691:0:2404:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695849.334692:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695849.334692:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695849.334695:0:2404:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446667480 x1874531139519488/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695849 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695849.334710:0:2404:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446667480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:608c526c-eb6e-4a54-80a5-8969828241eb:2404:1874531139519488:192.168.201.140@tcp:400:kworker.0 00000100:00000001:1.0:1787695849.334713:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695849.334715:0:2404:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695849.334718:0:2404:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446667480 x1874531139519488/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695849 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695849.334723:0:2404:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695849.334725:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695849.334728:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b144667d600. 02000000:00000001:1.0:1787695849.334732:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695849.334734:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695849.334736:0:2404:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=12 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695849.334737:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695849.334739:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1445678d00. 02000000:00000001:1.0:1787695849.334742:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695849.334744:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695849.334746:0:2404:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446667480. 00000100:00000001:1.0:1787695849.334749:0:2404:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695849.334750:0:2404:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695849.334752:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695849.398456:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695849.398469:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695849.398471:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695849.398475:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695849.398477:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695849.398481:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695849.398483:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695849.398485:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695849.398487:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695849.846451:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695849.846452:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695849.846467:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695849.846471:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695849.846474:0:2403:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695849.846474:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695849.846478:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695849.846479:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695849.846484:0:2403:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787695849.846485:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695849.846485:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695849.846492:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695849.846494:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695849.846496:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695849.846501:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695849.846504:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000400:0.0:1787695849.846506:0:2403:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695764/real 1787695764] req@ffff8b1446664700 x1874531139519360/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695849 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695849.846551:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695849.846554:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695849.846559:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617baa8. 00000400:00000200:0.0:1787695849.846567:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96a18 00000400:00000010:0.0:1787695849.846569:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96a18. 00000100:00000001:0.0:1787695849.846573:0:2403:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695849.846578:0:2403:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446664700 x1874531139519360/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695849 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695849.846593:0:2403:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446664700 x1874531139519360/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695849 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695849.846600:0:2403:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695849.846601:0:2403:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695849.846604:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695849.846606:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695849.846609:0:2403:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446664700 x1874531139519360/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695849 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695849.846615:0:2403:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:0.0:1787695849.846617:0:2403:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b1458081800 already not connected (conn 28, was 15): DISCONN 00000100:00000001:0.0:1787695849.846620:0:2403:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:0.0:1787695849.846621:0:2403:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695849.846628:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695849.846629:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=7 : 7 : 7) 00000100:00000001:0.0:1787695849.846631:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695849.846633:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695849.846636:0:2403:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695849.846638:0:2403:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695849.846640:0:2403:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695849.846643:0:2403:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446664700 x1874531139519360/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695849 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695849.846649:0:2403:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:0.0:1787695849.846656:0:2403:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695849.846657:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695849.846658:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695849.846662:0:2403:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446664700 x1874531139519360/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695849 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695849.846668:0:2403:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446664700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2403:1874531139519360:192.168.201.140@tcp:400:kworker.0 00000100:00000001:0.0:1787695849.846671:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695849.846674:0:2403:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695849.846676:0:2403:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446664700 x1874531139519360/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695849 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695849.846681:0:2403:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695849.846684:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695849.846688:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1451d64e00. 02000000:00000001:0.0:1787695849.846693:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695849.846696:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695849.846697:0:2403:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=11 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695849.846699:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695849.846702:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1445678700. 02000000:00000001:0.0:1787695849.846706:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695849.846708:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695849.846715:0:2403:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446664700. 00000100:00000001:0.0:1787695849.846718:0:2403:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695849.846719:0:2403:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695849.846722:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695850.038516:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695850.038544:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695850.038551:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695850.038552:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695850.038557:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695850.038558:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695850.038563:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695850.038564:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695850.038568:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695850.038569:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695850.038573:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695850.038574:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695850.038654:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695850.038660:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695850.038663:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695850.422406:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695850.422412:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695850.422413:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695850.422418:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695850.422420:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695850.422423:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695850.422425:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695850.422427:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695850.422429:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695850.870457:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695850.870457:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695850.870465:0:2405:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787695850.870465:0:2406:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000400:3.0:1787695850.870482:0:2406:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695765/real 1787695765] req@ffff8b1446605c00 x1874531139519744/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695850 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:2.0:1787695850.870500:0:2405:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695765/real 1787695765] req@ffff8b1446605500 x1874531139519616/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695850 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787695850.870512:0:2406:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787695850.870515:0:2406:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1787695850.870518:0:2406:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14456d1370. 00000400:00000200:3.0:1787695850.870525:0:2406:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b145078a550 00000400:00000010:3.0:1787695850.870529:0:2406:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b145078a550. 00000100:00000001:3.0:1787695850.870535:0:2406:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000001:2.0:1787695850.870541:0:2405:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000200:3.0:1787695850.870541:0:2406:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446605c00 x1874531139519744/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695850 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695850.870545:0:2405:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695850.870550:0:2405:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752b688. 00000100:00000200:3.0:1787695850.870554:0:2406:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446605c00 x1874531139519744/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695850 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787695850.870557:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695850.870562:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:3.0:1787695850.870568:0:2406:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695850.870569:0:2405:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787695850.870571:0:2406:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695850.870574:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695850.870577:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1787695850.870578:0:2405:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446605500 x1874531139519616/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695850 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695850.870583:0:2406:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446605c00 x1874531139519744/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695850 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695850.870609:0:2405:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446605500 x1874531139519616/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695850 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695850.870629:0:2406:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00000001:2.0:1787695850.870632:0:2405:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:3.0:1787695850.870632:0:2406:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b144515c000 already not connected (conn 24, was 11): DISCONN 00000100:00000001:2.0:1787695850.870636:0:2405:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695850.870637:0:2406:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:3.0:1787695850.870639:0:2406:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695850.870642:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695850.870645:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695850.870647:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695850.870650:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695850.870651:0:2405:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446605500 x1874531139519616/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695850 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695850.870654:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695850.870658:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695850.870663:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695850.870664:0:2405:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00000001:3.0:1787695850.870667:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1787695850.870669:0:2405:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b1458081800 already not connected (conn 28, was 15): DISCONN 00000100:00000001:3.0:1787695850.870671:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695850.870675:0:2405:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:2.0:1787695850.870677:0:2405:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695850.870682:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:3.0:1787695850.870682:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446605c00 x1874531139519744/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695850 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695850.870685:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695850.870690:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695850.870693:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695850.870696:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695850.870701:0:2405:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695850.870702:0:2406:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:2.0:1787695850.870703:0:2405:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695850.870709:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695850.870710:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446605500 x1874531139519616/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695850 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695850.870712:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695850.870714:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695850.870722:0:2405:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000040:3.0:1787695850.870723:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446605c00 x1874531139519744/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695850 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695850.870731:0:2406:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446605c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:608c526c-eb6e-4a54-80a5-8969828241eb:2406:1874531139519744:192.168.201.140@tcp:400:kworker.0 00000100:00000001:2.0:1787695850.870733:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695850.870736:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695850.870737:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695850.870739:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695850.870742:0:2406:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695850.870746:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446605500 x1874531139519616/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695850 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695850.870748:0:2406:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446605c00 x1874531139519744/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695850 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695850.870772:0:2405:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446605500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2405:1874531139519616:192.168.201.140@tcp:400:kworker.0 00000100:00000001:3.0:1787695850.870772:0:2406:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787695850.870777:0:2406:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1787695850.870779:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695850.870781:0:2406:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1460531400. 00000100:00000001:2.0:1787695850.870782:0:2405:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695850.870788:0:2405:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446605500 x1874531139519616/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695850 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695850.870788:0:2406:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787695850.870792:0:2406:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1787695850.870795:0:2406:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=11 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000001:2.0:1787695850.870796:0:2405:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695850.870798:0:2405:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695850.870800:0:2405:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b14484ac200. 02000000:00000001:2.0:1787695850.870805:0:2405:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787695850.870806:0:2406:0:(genops.c:1230:class_import_put()) Process leaving 00000020:00000001:2.0:1787695850.870808:0:2405:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695850.870810:0:2405:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=10 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000010:3.0:1787695850.870810:0:2406:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1458095c00. 00000020:00000001:2.0:1787695850.870815:0:2405:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695850.870817:0:2405:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1458095d00. 02000000:00000001:3.0:1787695850.870817:0:2406:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787695850.870820:0:2406:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1787695850.870822:0:2405:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000010:3.0:1787695850.870823:0:2406:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446605c00. 02000000:00000001:2.0:1787695850.870825:0:2405:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695850.870828:0:2405:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446605500. 00000100:00000001:3.0:1787695850.870828:0:2406:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787695850.870832:0:2406:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695850.870833:0:2405:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787695850.870834:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695850.870835:0:2405:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695850.870839:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695851.062494:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695851.062501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695851.062526:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695851.062536:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695851.062541:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695851.062543:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695851.062547:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695851.062548:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695851.062551:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695851.062552:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695851.062556:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695851.062557:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695851.062611:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695851.062614:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695851.062617:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787695851.383198:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695851.383211:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787695851.383215:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695851.383219:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695851.383225:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695851.383234:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3739 00000400:00000200:3.0:1787695851.383254:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695851.383263:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695851.383267:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695851.383271:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695851.383275:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695851.383278:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695851.383281:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695851.383285:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695851.383287:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:3.0:1787695851.383291:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695851.383295:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787695851.383297:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787695851.383300:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787695851.383303:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695851.383305:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695851.383319:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695851.383321:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695851.383325:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787695851.383329:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695851.383332:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695851.383335:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695851.383339:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695851.383343:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695851.383345:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787695851.383347:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695851.383352:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7480. 00000020:00000040:3.0:1787695851.383355:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787695851.383359:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695851.383361:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695851.383386:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695851.383389:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695851.383394:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695851.383404:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674800. 00000100:00000001:3.0:1787695851.383411:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695851.383423:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7480 x1874531139533056/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695851.383435:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695851.383440:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695851.383445:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043daeda0] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7480 x1874531139533056/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695851.383468:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695851.383472:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695851.383474:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695851.383475:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695851.383476:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695851.383479:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695851.383487:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3739 00000400:00000200:3.0:1787695851.383491:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695851.383494:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695851.383496:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695851.383498:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695851.383500:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695851.383503:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695851.383506:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695851.383509:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695851.383510:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000100:00000001:0.0:1787695851.383512:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787695851.383513:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787695851.383518:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1787695851.383519:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000100:00000001:0.0:1787695851.383522:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1787695851.383524:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:3.0:1787695851.383530:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:0.0:1787695851.383531:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7480 x1874531139533056/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695851.383538:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695851.383544:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695851.383546:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1787695851.383549:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695851.383552:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695851.383556:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695851.383557:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 02000000:00000001:0.0:1787695851.383561:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1787695851.383564:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000001:0.0:1787695851.383565:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695851.383567:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695851.383569:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695851.383570:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:0.0:1787695851.383575:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139533056:192.168.201.140@tcp:250:kworker.0 02000000:00000001:3.0:1787695851.383576:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695851.383581:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787695851.383581:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695851.383585:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787695851.383586:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695851.383589:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787695851.383590:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787695851.383592:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695851.383597:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450774400. 00000008:00000001:3.0:1787695851.383597:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695851.383602:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695851.383602:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695851.383613:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000010:3.0:1787695851.383613:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4e00. 00000400:00000010:0.0:1787695851.383619:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617baa8. 00000020:00000040:3.0:1787695851.383620:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695851.383626:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:0.0:1787695851.383627:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96a18. 00000100:00000200:0.0:1787695851.383634:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139533056, portal 25 02000000:00000001:3.0:1787695851.383635:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.383637:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787695851.383640:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787695851.383641:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930157184 : -128555779394432 : ffff8b14466f7480) 02000000:00000001:3.0:1787695851.383645:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695851.383648:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695851.383652:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 00000100:00000040:0.0:1787695851.383657:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7480 x1874531139533056/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695851.383658:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695851.383667:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4e00 x1874531139533184/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695851.383669:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695851.383676:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695851.383679:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000010:0.0:1787695851.383681:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00000001:3.0:1787695851.383683:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695851.383689:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139533056, offset 0 00000100:00000040:3.0:1787695851.383689:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcd77cd6] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4e00 x1874531139533184/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695851.383700:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695851.383701:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00080000:3.0:1787695851.383704:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695851.383707:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695851.383709:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695851.383710:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695851.383713:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:0.0:1787695851.383715:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:3.0:1787695851.383718:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3739 00000400:00000200:0.0:1787695851.383725:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:3.0:1787695851.383725:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695851.383732:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000200:0.0:1787695851.383734:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:3.0:1787695851.383735:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695851.383740:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695851.383742:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:3.0:1787695851.383743:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1787695851.383748:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000040:3.0:1787695851.383749:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695851.383753:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:0.0:1787695851.383755:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000020:00000001:3.0:1787695851.383756:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695851.383759:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000400:00000200:0.0:1787695851.383762:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16974:16974:256:4294967295] 192.168.201.140@tcp LPNI seq info [16973:16973:8:4294967295] 00000020:00000001:3.0:1787695851.383762:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695851.383765:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:3.0:1787695851.383769:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000400:00000200:0.0:1787695851.383770:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:3.0:1787695851.383776:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:3.0:1787695851.383783:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695851.383787:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:3.0:1787695851.383788:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695851.383791:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695851.383792:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695851.383794:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:3.0:1787695851.383800:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695851.383802:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695851.383805:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:0.0:1787695851.383807:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695851.383809:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695851.383812:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1787695851.383813:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfcd00. 00000100:00000001:3.0:1787695851.383814:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695851.383818:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695851.383821:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1787695851.383824:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787695851.383824:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695851.383828:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7100. 00000800:00000200:0.0:1787695851.383829:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 18 secs later) 00000020:00000040:3.0:1787695851.383832:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=11 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695851.383835:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695851.383838:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.1:1787695851.383838:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787695851.383842:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695851.383845:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:0.1:1787695851.383845:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 18 secs later) 02000000:00000001:3.0:1787695851.383847:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695851.383852:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676000. 00000800:00000100:0.0:1787695851.383853:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695851.383858:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfcd00. 00000100:00000001:3.0:1787695851.383858:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695851.383865:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7100 x1874531139533312/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1787695851.383867:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695851.383876:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1787695851.383881:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:0.0:1787695851.383884:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000400:00000010:0.0:1787695851.383887:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000001:3.0:1787695851.383887:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.383895:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1787695851.383897:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000004e8abd7] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7100 x1874531139533312/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695851.383902:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7480 x1874531139533056/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.383917:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695851.383920:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7480 x1874531139533056/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695876 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695851.383929:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.383931:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.383932:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695851.383937:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695851.383940:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:0.0:1787695851.383942:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695851.383947:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298423097 00000100:00080000:3.0:1787695851.383948:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787695851.383953:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695851.383955:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695851.383956:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298417977 00000100:00000001:0.0:1787695851.383963:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298423097 00000100:00000001:3.0:1787695851.383963:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787695851.383969:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298417977 00000100:00080000:3.0:1787695851.383970:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3739 00000100:00000001:0.0:1787695851.383974:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695851.383978:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1787695851.383979:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695851.383981:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000400:00000200:3.0:1787695851.383998:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695851.384001:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787695851.384003:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695851/real 1787695851] req@ffff8b14466f7480 x1874531139533056/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695851.384006:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695851.384012:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695851.384015:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695851.384018:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695851.384022:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695851.384024:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:3.0:1787695851.384025:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695851.384027:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:3.0:1787695851.384030:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:3.0:1787695851.384034:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:3.0:1787695851.384041:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695851.384046:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:0.0:1787695851.384047:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695851.384050:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:3.0:1787695851.384051:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000010:0.0:1787695851.384052:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617baa8. 00000100:00000001:3.0:1787695851.384057:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:0.0:1787695851.384059:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96a18 00000400:00000010:0.0:1787695851.384062:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96a18. 00000100:00000001:0.0:1787695851.384064:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695851.384069:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7480 x1874531139533056/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1787695851.384069:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:3.0:1787695851.384079:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695851.384084:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695851.384085:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7480 x1874531139533056/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695851.384091:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787695851.384096:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:3.0:1787695851.384097:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695851.384098:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.384103:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1787695851.384104:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.384105:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695851.384107:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:0.0:1787695851.384110:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7480 x1874531139533056/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787695851.384117:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787695851.384119:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1787695851.384123:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695851.384127:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7480 x1874531139533056/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695851.384127:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695851.384132:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7800. 00000020:00000040:3.0:1787695851.384136:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000100:00000001:0.0:1787695851.384141:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1787695851.384142:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695851.384145:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.384146:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695851.384148:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787695851.384149:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1787695851.384151:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787695851.384152:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787695851.384155:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.384156:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695851.384160:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000010:3.0:1787695851.384161:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 00000100:00080000:0.0:1787695851.384163:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787695851.384166:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695851.384168:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695851.384173:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787695851.384176:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7800 x1874531139533440/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:0.0:1787695851.384180:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7480 x1874531139533056/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695851.384185:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695851.384189:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695851.384196:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7c57981] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7800 x1874531139533440/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695851.384205:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139533056:192.168.201.140@tcp:250:kworker.0 00000100:00000001:3.0:1787695851.384207:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.384210:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695851.384211:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695851.384215:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1787695851.384216:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1787695851.384218:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:0.0:1787695851.384220:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7480 x1874531139533056/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695851.384221:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695851.384225:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695851.384230:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3739 00000100:00000001:0.0:1787695851.384233:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695851.384237:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:3.0:1787695851.384240:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695851.384244:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 02000000:00000010:0.0:1787695851.384246:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450774400. 00000100:00080000:3.0:1787695851.384247:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 02000000:00000001:0.0:1787695851.384252:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:3.0:1787695851.384254:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000020:00000001:0.0:1787695851.384255:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:3.0:1787695851.384259:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000040:0.0:1787695851.384261:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000100:00000040:3.0:1787695851.384265:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000020:00000001:0.0:1787695851.384266:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695851.384269:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674800. 02000000:00000001:0.0:1787695851.384272:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695851.384274:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695851.384278:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7480. 00000100:00000001:3.0:1787695851.384279:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695851.384281:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695851.384283:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.384286:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787695851.384286:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695851.384291:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000100:00000001:0.0:1787695851.384294:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:3.0:1787695851.384294:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695851.384296:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000100:00000001:0.0:1787695851.384297:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695851.384301:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000001:0.0:1787695851.384302:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787695851.384305:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000001:0.0:1787695851.384306:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695851.384309:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1787695851.384312:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695851.384316:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695851.384317:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4e00 x1874531139533184/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695851.384320:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695851.384324:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695851.384329:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00000001:0.0:1787695851.384334:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1787695851.384335:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695851.384337:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695851.384339:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695851.384341:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787695851.384342:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787695851.384345:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695851.384347:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695851.384348:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695851.384352:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695851.384355:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139533184:192.168.201.140@tcp:38:kworker.0 00000100:00000001:3.0:1787695851.384356:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695851.384358:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787695851.384360:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695851.384383:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695851.384388:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5c00. 00000020:00000040:3.0:1787695851.384390:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:3.0:1787695851.384392:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695851.384393:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695851.384394:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695851.384396:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695851.384398:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695851.384401:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675400. 00000100:00000001:3.0:1787695851.384404:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695851.384411:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5c00 x1874531139533568/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695851.384413:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695851.384417:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695851.384420:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695851.384423:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695851.384427:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695851.384430:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695851.384438:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695851.384445:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617baa8. 00000100:00000001:3.0:1787695851.384446:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695851.384451:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695851.384453:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96a18. 00000100:00000200:0.0:1787695851.384459:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139533184, portal 10 00000100:00000040:3.0:1787695851.384459:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005bfbcee1] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5c00 x1874531139533568/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.384465:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695851.384469:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930147328 : -128555779404288 : ffff8b14466f4e00) 00000100:00000001:3.0:1787695851.384475:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695851.384478:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1787695851.384483:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4e00 x1874531139533184/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695851.384483:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695851.384486:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695851.384488:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695851.384492:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695851.384498:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3739 00000100:00000001:0.0:1787695851.384501:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695851.384507:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000200:3.0:1787695851.384507:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695851.384513:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000400:00000200:3.0:1787695851.384516:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000200:0.0:1787695851.384518:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139533184, offset 0 00000100:00080000:3.0:1787695851.384522:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000400:00000200:0.0:1787695851.384526:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695851.384530:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695851.384533:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695851.384537:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:0.0:1787695851.384540:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000040:3.0:1787695851.384545:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695851.384546:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1787695851.384551:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:0.0:1787695851.384553:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000020:00000001:3.0:1787695851.384556:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:0.0:1787695851.384558:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000020:00000040:3.0:1787695851.384559:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000400:00000200:0.0:1787695851.384563:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000020:00000001:3.0:1787695851.384564:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695851.384567:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000400:00000200:0.0:1787695851.384568:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16975:16975:256:4294967295] 192.168.201.140@tcp LPNI seq info [16974:16974:8:4294967295] 00000020:00000001:3.0:1787695851.384570:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000020:00000001:3.0:1787695851.384576:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000200:0.0:1787695851.384578:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695851.384585:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:0.0:1787695851.384589:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:3.0:1787695851.384591:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695851.384598:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000200:0.0:1787695851.384599:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695851.384604:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000010:0.0:1787695851.384605:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfcd00. 00000800:00000200:0.0:1787695851.384609:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000020:00000040:3.0:1787695851.384618:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000800:00000200:0.0:1787695851.384619:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 18 secs later) 00000100:00080000:3.0:1787695851.384628:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000800:00000200:0.1:1787695851.384629:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695851.384634:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 18 secs later) 00000100:00000001:3.0:1787695851.384635:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695851.384639:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:0.0:1787695851.384642:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695851.384643:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:0.0:1787695851.384645:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfcd00. 02000000:00000001:3.0:1787695851.384647:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695851.384650:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1787695851.384650:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695851.384654:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000200:0.0:1787695851.384655:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000008:00000001:3.0:1787695851.384658:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695851.384660:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000100:00000001:3.0:1787695851.384661:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695851.384665:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000010:3.0:1787695851.384667:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6680. 00000100:00000001:0.0:1787695851.384669:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:3.0:1787695851.384671:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000100:00000200:0.0:1787695851.384675:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4e00 x1874531139533184/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695851.384676:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695851.384678:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695851.384682:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695851.384684:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695851.384687:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.384689:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000010:3.0:1787695851.384692:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 00000100:00000040:0.0:1787695851.384693:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4e00 x1874531139533184/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695876 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695851.384697:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.384703:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.384707:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:3.0:1787695851.384707:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6680 x1874531139533696/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695851.384710:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695851.384716:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000100:00000001:3.0:1787695851.384718:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695851.384721:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695851.384722:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695851.384726:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298412999 00000100:00000040:3.0:1787695851.384727:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f6d2abf9] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6680 x1874531139533696/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.384731:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298423097 00000100:00000001:0.0:1787695851.384736:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298412999 00000100:00000001:0.0:1787695851.384741:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298423097 00000100:00000001:3.0:1787695851.384741:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695851.384745:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695851.384746:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695851.384751:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695851.384753:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787695851.384754:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787695851.384755:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695851.384758:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787695851.384762:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695851/real 1787695851] req@ffff8b14466f4e00 x1874531139533184/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695851.384763:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000001:0.0:1787695851.384769:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695851.384772:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1787695851.384776:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3739 00000400:00000010:0.0:1787695851.384781:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617baa8. 00000400:00000200:3.0:1787695851.384785:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695851.384786:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96a18 00000400:00000010:0.0:1787695851.384789:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96a18. 00000100:00000001:0.0:1787695851.384793:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:3.0:1787695851.384796:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000200:0.0:1787695851.384800:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4e00 x1874531139533184/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695851.384801:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695851.384810:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000200:0.0:1787695851.384814:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4e00 x1874531139533184/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695851.384815:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695851.384821:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695851.384823:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695851.384826:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695851.384827:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695851.384828:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695851.384831:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787695851.384834:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00100000:0.0:1787695851.384837:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4e00 x1874531139533184/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1787695851.384838:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000020:00000001:3.0:1787695851.384842:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695851.384846:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000100:00000001:0.0:1787695851.384847:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695851.384853:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4e00 x1874531139533184/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695851.384853:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000020:00000001:3.0:1787695851.384860:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:0.0:1787695851.384865:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695851.384869:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695851.384870:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695851.384871:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695851.384874:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695851.384876:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695851.384877:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695851.384879:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695851.384882:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000040:3.0:1787695851.384883:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695851.384887:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00080000:0.0:1787695851.384888:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000020:00000040:3.0:1787695851.384891:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00000001:0.0:1787695851.384893:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:3.0:1787695851.384897:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000040:0.0:1787695851.384899:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4e00 x1874531139533184/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695851.384900:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695851.384904:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695851.384908:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1787695851.384911:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139533184:192.168.201.140@tcp:38:kworker.0 02000000:00000001:3.0:1787695851.384912:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695851.384914:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695851.384916:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695851.384920:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.384921:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787695851.384922:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695851.384926:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7100 x1874531139533312/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1787695851.384927:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5880. 00000020:00000040:3.0:1787695851.384932:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=12 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:3.0:1787695851.384937:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787695851.384938:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695851.384940:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695851.384941:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695851.384943:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695851.384945:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695851.384946:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695851.384950:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:0.0:1787695851.384951:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139533312:192.168.201.140@tcp:38:kworker.0 02000000:00000001:3.0:1787695851.384953:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787695851.384957:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787695851.384957:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695851.384960:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695851.384963:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695851.384963:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 02000000:00000001:0.0:1787695851.384968:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787695851.384970:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695851.384972:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450777000. 02000000:00000001:0.0:1787695851.384976:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695851.384977:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5880 x1874531139533824/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:0.0:1787695851.384983:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695851.385000:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617baa8. 00000400:00000010:0.0:1787695851.385005:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96a18. 00000100:00000001:3.0:1787695851.385006:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000200:0.0:1787695851.385011:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139533312, portal 10 00000100:00000001:3.0:1787695851.385011:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.385016:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:3.0:1787695851.385017:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000827e7dbe] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5880 x1874531139533824/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385018:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930156288 : -128555779395328 : ffff8b14466f7100) 00000100:00000040:0.0:1787695851.385029:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7100 x1874531139533312/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695851.385029:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695851.385032:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787695851.385039:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:0.0:1787695851.385044:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1787695851.385046:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1787695851.385047:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695851.385051:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00000200:0.0:1787695851.385055:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139533312, offset 0 00000100:00000040:3.0:1787695851.385055:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:0.0:1787695851.385061:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695851.385065:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695851.385065:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3749) 00000400:00000200:0.0:1787695851.385071:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695851.385074:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695851.385081:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695851.385083:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695851.385085:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695851.385087:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16976:16976:256:4294967295] 192.168.201.140@tcp LPNI seq info [16975:16975:8:4294967295] 00000400:00000200:0.0:1787695851.385090:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695851.385095:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695851.385098:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695851.385101:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfcd00. 00000800:00000200:0.0:1787695851.385103:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695851.385105:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 18 secs later) 00000800:00000200:0.1:1787695851.385108:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695851.385110:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 18 secs later) 00000800:00000100:0.0:1787695851.385113:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695851.385114:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfcd00. 00000400:00000200:0.0:1787695851.385116:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695851.385119:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695851.385121:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000400:00000010:0.0:1787695851.385122:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000001:0.0:1787695851.385124:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695851.385126:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7100 x1874531139533312/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385132:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695851.385134:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7100 x1874531139533312/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695876 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385138:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.385139:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695851.385142:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695851.385144:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:0.0:1787695851.385146:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695851.385147:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298411847 00000100:00000001:0.0:1787695851.385149:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298423097 00000100:00000001:0.0:1787695851.385150:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298411847 00000100:00000001:0.0:1787695851.385155:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298423097 00000100:00000001:0.0:1787695851.385156:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695851.385158:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.385159:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695851.385162:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695851/real 1787695851] req@ffff8b14466f7100 x1874531139533312/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695851.385167:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695851.385168:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695851.385169:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617baa8. 00000400:00000200:0.0:1787695851.385171:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96a18 00000400:00000010:0.0:1787695851.385172:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96a18. 00000100:00000001:0.0:1787695851.385173:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695851.385175:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7100 x1874531139533312/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695851.385181:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7100 x1874531139533312/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385185:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695851.385185:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.385186:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695851.385187:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695851.385189:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7100 x1874531139533312/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385193:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695851.385196:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7100 x1874531139533312/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385200:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695851.385201:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.385202:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695851.385203:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.385207:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695851.385208:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695851.385210:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695851.385212:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695851.385214:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695851.385216:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7100 x1874531139533312/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695851.385221:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139533312:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695851.385223:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695851.385226:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7800 x1874531139533440/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385230:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695851.385231:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695851.385233:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695851.385234:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695851.385235:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695851.385237:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139533440:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695851.385239:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695851.385240:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695851.385241:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695851.385242:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695851.385244:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450775000. 02000000:00000001:0.0:1787695851.385246:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695851.385249:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695851.385250:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617baa8. 00000400:00000010:0.0:1787695851.385253:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96a18. 00000100:00000200:0.0:1787695851.385255:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139533440, portal 4 00000100:00000001:0.0:1787695851.385256:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695851.385257:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930158080 : -128555779393536 : ffff8b14466f7800) 00000100:00000040:0.0:1787695851.385261:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7800 x1874531139533440/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385268:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695851.385270:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695851.385271:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00000200:0.0:1787695851.385273:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139533440, offset 0 00000400:00000200:0.0:1787695851.385276:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695851.385278:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695851.385280:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695851.385282:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695851.385284:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695851.385286:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695851.385287:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695851.385290:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16977:16977:256:4294967295] 192.168.201.140@tcp LPNI seq info [16976:16976:8:4294967295] 00000400:00000200:0.0:1787695851.385292:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695851.385296:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695851.385299:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695851.385301:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfcd00. 00000800:00000200:0.0:1787695851.385303:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695851.385305:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 18 secs later) 00000800:00000200:0.1:1787695851.385307:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695851.385309:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 18 secs later) 00000800:00000100:0.0:1787695851.385315:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695851.385316:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfcd00. 00000400:00000200:0.0:1787695851.385318:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695851.385321:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695851.385323:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000400:00000010:0.0:1787695851.385324:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000001:0.0:1787695851.385326:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695851.385328:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7800 x1874531139533440/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385333:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695851.385335:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7800 x1874531139533440/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695876 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385348:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.385349:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695851.385351:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695851.385353:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:0.0:1787695851.385356:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695851.385357:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298411877 00000100:00000001:0.0:1787695851.385359:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298423097 00000100:00000001:0.0:1787695851.385360:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298411877 00000100:00000001:0.0:1787695851.385362:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298423097 00000100:00000001:0.0:1787695851.385377:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695851.385379:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.385380:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695851.385383:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695851/real 1787695851] req@ffff8b14466f7800 x1874531139533440/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695851.385390:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695851.385391:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695851.385393:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617baa8. 00000400:00000200:0.0:1787695851.385395:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96a18 00000400:00000010:0.0:1787695851.385397:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96a18. 00000100:00000001:0.0:1787695851.385398:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695851.385401:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7800 x1874531139533440/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695851.385407:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7800 x1874531139533440/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385410:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695851.385411:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.385412:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695851.385413:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695851.385415:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7800 x1874531139533440/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385423:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695851.385426:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7800 x1874531139533440/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385430:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695851.385432:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.385433:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695851.385434:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.385435:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695851.385436:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695851.385437:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695851.385440:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695851.385441:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695851.385445:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7800 x1874531139533440/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695851.385454:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139533440:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695851.385458:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.385460:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695851.385463:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4e00 x1874531139533184/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385470:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695851.385472:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695851.385474:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695851.385477:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695851.385479:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695851.385480:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695851.385483:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695851.385485:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:0.0:1787695851.385493:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695851.385494:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695851.385496:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4e00. 00000100:00000001:0.0:1787695851.385498:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695851.385499:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.385500:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695851.385504:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7100 x1874531139533312/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385508:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695851.385509:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695851.385510:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450777000. 02000000:00000001:0.0:1787695851.385512:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695851.385513:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695851.385513:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=10 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:0.0:1787695851.385515:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695851.385516:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676000. 02000000:00000001:0.0:1787695851.385518:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695851.385518:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695851.385519:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7100. 00000100:00000001:0.0:1787695851.385520:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695851.385521:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.385522:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695851.385524:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7800 x1874531139533440/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385528:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695851.385529:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695851.385530:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450775000. 02000000:00000001:0.0:1787695851.385531:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695851.385532:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695851.385532:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:0.0:1787695851.385534:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695851.385534:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:0.0:1787695851.385537:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695851.385538:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695851.385539:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7800. 00000100:00000001:0.0:1787695851.385540:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695851.385543:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.385545:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.385548:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695851.385549:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.385550:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695851.385551:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695851.385552:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695851.385555:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5c00 x1874531139533568/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385559:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695851.385560:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695851.385562:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695851.385563:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695851.385565:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695851.385566:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139533568:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695851.385568:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695851.385570:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695851.385571:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695851.385572:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695851.385574:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450775000. 02000000:00000001:0.0:1787695851.385575:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695851.385578:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695851.385580:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617baa8. 00000400:00000010:0.0:1787695851.385582:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96a18. 00000100:00000200:0.0:1787695851.385584:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139533568, portal 4 00000100:00000001:0.0:1787695851.385585:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695851.385587:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150912 : -128555779400704 : ffff8b14466f5c00) 00000100:00000040:0.0:1787695851.385591:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5c00 x1874531139533568/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385595:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695851.385596:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695851.385598:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00000200:0.0:1787695851.385599:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139533568, offset 0 00000400:00000200:0.0:1787695851.385602:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695851.385608:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695851.385610:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695851.385612:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695851.385614:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695851.385616:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695851.385617:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695851.385619:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16978:16978:256:4294967295] 192.168.201.140@tcp LPNI seq info [16977:16977:8:4294967295] 00000400:00000200:0.0:1787695851.385622:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695851.385627:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695851.385629:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695851.385632:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfcd00. 00000800:00000200:0.0:1787695851.385634:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695851.385636:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 18 secs later) 00000800:00000200:0.1:1787695851.385639:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695851.385641:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 18 secs later) 00000800:00000100:0.0:1787695851.385643:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695851.385645:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfcd00. 00000400:00000200:0.0:1787695851.385647:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695851.385649:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695851.385651:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000400:00000010:0.0:1787695851.385652:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000001:0.0:1787695851.385654:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695851.385656:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5c00 x1874531139533568/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385661:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695851.385663:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5c00 x1874531139533568/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695876 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385668:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.385669:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695851.385671:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695851.385672:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:0.0:1787695851.385675:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695851.385679:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298411550 00000100:00000001:0.0:1787695851.385681:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298423097 00000100:00000001:0.0:1787695851.385682:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298411550 00000100:00000001:0.0:1787695851.385684:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298423097 00000100:00000001:0.0:1787695851.385685:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695851.385686:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.385687:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695851.385690:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695851/real 1787695851] req@ffff8b14466f5c00 x1874531139533568/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695851.385695:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695851.385697:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695851.385698:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617baa8. 00000400:00000200:0.0:1787695851.385699:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96a18 00000400:00000010:0.0:1787695851.385700:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96a18. 00000100:00000001:0.0:1787695851.385702:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695851.385704:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5c00 x1874531139533568/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695851.385709:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5c00 x1874531139533568/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385713:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695851.385714:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.385715:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695851.385716:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695851.385718:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5c00 x1874531139533568/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385722:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695851.385724:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5c00 x1874531139533568/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385728:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695851.385732:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.385733:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695851.385734:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.385736:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695851.385737:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695851.385738:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695851.385741:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695851.385742:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695851.385744:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5c00 x1874531139533568/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695851.385749:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139533568:192.168.201.140@tcp:8:kworker.0 00000100:00000001:0.0:1787695851.385752:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695851.385754:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6680 x1874531139533696/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385758:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695851.385759:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695851.385760:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695851.385761:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695851.385762:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695851.385764:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139533696:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695851.385766:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695851.385767:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695851.385768:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695851.385769:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695851.385771:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450777000. 02000000:00000001:0.0:1787695851.385772:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695851.385775:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695851.385776:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617baa8. 00000400:00000010:0.0:1787695851.385778:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96a18. 00000100:00000200:0.0:1787695851.385780:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139533696, portal 10 00000100:00000001:0.0:1787695851.385785:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695851.385786:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930153600 : -128555779398016 : ffff8b14466f6680) 00000100:00000040:0.0:1787695851.385791:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6680 x1874531139533696/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385795:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695851.385797:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695851.385798:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00000200:0.0:1787695851.385800:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139533696, offset 0 00000400:00000200:0.0:1787695851.385802:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695851.385805:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695851.385807:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695851.385808:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695851.385810:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695851.385811:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695851.385813:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695851.385815:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16979:16979:256:4294967295] 192.168.201.140@tcp LPNI seq info [16978:16978:8:4294967295] 00000400:00000200:0.0:1787695851.385818:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695851.385822:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695851.385825:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695851.385827:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfcd00. 00000800:00000200:0.0:1787695851.385829:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695851.385831:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 18 secs later) 00000800:00000200:0.1:1787695851.385833:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695851.385835:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 18 secs later) 00000800:00000100:0.0:1787695851.385838:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695851.385839:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfcd00. 00000400:00000200:0.0:1787695851.385841:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695851.385843:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695851.385845:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000400:00000010:0.0:1787695851.385846:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000001:0.0:1787695851.385847:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695851.385849:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6680 x1874531139533696/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385862:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695851.385864:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6680 x1874531139533696/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695876 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385868:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.385869:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695851.385871:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695851.385874:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:0.0:1787695851.385876:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695851.385877:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298411736 00000100:00000001:0.0:1787695851.385879:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298423097 00000100:00000001:0.0:1787695851.385880:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298411736 00000100:00000001:0.0:1787695851.385882:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298423097 00000100:00000001:0.0:1787695851.385883:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695851.385884:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.385885:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695851.385888:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695851/real 1787695851] req@ffff8b14466f6680 x1874531139533696/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695851.385892:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695851.385893:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695851.385894:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617baa8. 00000400:00000200:0.0:1787695851.385895:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96a18 00000400:00000010:0.0:1787695851.385896:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96a18. 00000100:00000001:0.0:1787695851.385897:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695851.385899:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6680 x1874531139533696/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695851.385904:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6680 x1874531139533696/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385908:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695851.385912:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.385913:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695851.385914:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695851.385917:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6680 x1874531139533696/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385921:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695851.385923:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6680 x1874531139533696/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385927:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695851.385929:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.385930:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695851.385930:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.385931:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695851.385932:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695851.385934:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695851.385936:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695851.385938:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695851.385940:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6680 x1874531139533696/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695851.385945:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139533696:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695851.385947:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695851.385949:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5880 x1874531139533824/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.385952:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695851.385953:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695851.385955:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787695851.385956:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695851.385957:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695851.385962:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139533824:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695851.385964:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787695851.385966:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695851.385967:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695851.385968:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1787695851.385970:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695851.385971:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695851.385973:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:0.0:1787695851.385974:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144617baa8. 00000400:00000010:0.0:1787695851.385976:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f96a18. 00000100:00000200:0.0:1787695851.385978:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139533824, portal 10 00000100:00000001:0.0:1787695851.385980:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695851.385981:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150016 : -128555779401600 : ffff8b14466f5880) 00000100:00000040:0.0:1787695851.385996:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5880 x1874531139533824/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.386001:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1787695851.386003:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:0.0:1787695851.386005:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1447f966e8. 00000100:00000200:0.0:1787695851.386007:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139533824, offset 0 00000400:00000200:0.0:1787695851.386009:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:0.0:1787695851.386012:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695851.386014:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:0.0:1787695851.386016:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1787695851.386018:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:0.0:1787695851.386019:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1787695851.386021:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:0.0:1787695851.386023:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16980:16980:256:4294967295] 192.168.201.140@tcp LPNI seq info [16979:16979:8:4294967295] 00000400:00000200:0.0:1787695851.386025:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695851.386030:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:0.0:1787695851.386032:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695851.386034:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1442cfcd00. 00000800:00000200:0.0:1787695851.386041:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.0:1787695851.386043:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 18 secs later) 00000800:00000200:0.1:1787695851.386045:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:0.1:1787695851.386047:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 18 secs later) 00000800:00000100:0.0:1787695851.386049:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:0.0:1787695851.386051:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1442cfcd00. 00000400:00000200:0.0:1787695851.386053:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:0.0:1787695851.386055:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:0.0:1787695851.386057:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f966e8 00000400:00000010:0.0:1787695851.386058:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f966e8. 00000100:00000001:0.0:1787695851.386060:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695851.386062:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5880 x1874531139533824/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695876 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.386068:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695851.386070:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5880 x1874531139533824/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695876 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.386074:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.386075:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787695851.386077:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695851.386079:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:0.0:1787695851.386081:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695851.386082:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298411714 00000100:00000001:0.0:1787695851.386084:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298423097 00000100:00000001:0.0:1787695851.386085:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298411714 00000100:00000001:0.0:1787695851.386087:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298423097 00000100:00000001:0.0:1787695851.386088:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695851.386089:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.386091:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787695851.386093:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695851/real 1787695851] req@ffff8b14466f5880 x1874531139533824/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787695851.386098:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695851.386101:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1787695851.386103:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617baa8. 00000400:00000200:0.0:1787695851.386104:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96a18 00000400:00000010:0.0:1787695851.386105:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96a18. 00000100:00000001:0.0:1787695851.386107:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1787695851.386109:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5880 x1874531139533824/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1787695851.386115:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5880 x1874531139533824/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.386119:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695851.386120:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.386121:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695851.386122:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695851.386124:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5880 x1874531139533824/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.386128:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695851.386130:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5880 x1874531139533824/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.386134:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695851.386135:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.386136:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695851.386137:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.386138:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695851.386139:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695851.386141:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695851.386143:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695851.386144:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:0.0:1787695851.386147:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5880 x1874531139533824/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787695851.386158:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139533824:192.168.201.140@tcp:38:kworker.0 00000100:00000001:0.0:1787695851.386161:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.386162:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695851.386164:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5c00 x1874531139533568/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.386168:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695851.386169:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695851.386170:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450775000. 02000000:00000001:0.0:1787695851.386172:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695851.386173:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695851.386174:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:0.0:1787695851.386175:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695851.386177:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675400. 02000000:00000001:0.0:1787695851.386179:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695851.386179:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695851.386181:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5c00. 00000100:00000001:0.0:1787695851.386182:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695851.386183:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.386184:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695851.386186:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6680 x1874531139533696/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.386190:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695851.386191:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695851.386191:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450777000. 02000000:00000001:0.0:1787695851.386193:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695851.386193:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695851.386194:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695851.386195:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695851.386196:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:0.0:1787695851.386197:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695851.386198:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695851.386199:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6680. 00000100:00000001:0.0:1787695851.386205:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695851.386205:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.386207:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695851.386209:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5880 x1874531139533824/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695876 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695851.386213:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695851.386214:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1787695851.386215:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1450774400. 02000000:00000001:0.0:1787695851.386216:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787695851.386217:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695851.386218:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=11 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:0.0:1787695851.386219:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:0.0:1787695851.386220:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:0.0:1787695851.386221:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787695851.386222:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1787695851.386223:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5880. 00000100:00000001:0.0:1787695851.386224:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787695851.386225:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.386226:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695851.386228:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695851.386229:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695851.386230:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695851.386231:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695851.446477:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695851.446485:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695851.446487:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695851.446494:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695851.446498:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695851.446501:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695851.446504:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695851.446508:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695851.446511:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695851.894473:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695851.894486:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695851.894489:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695851.894495:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695851.894497:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695851.894501:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695851.894506:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695851.894511:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695851.894522:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695851.894524:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695851.894529:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695851.894530:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695851.894556:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695851.894559:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695852.086458:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695852.086465:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695852.086470:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695852.086471:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695852.086475:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695852.086475:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695852.086478:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695852.086479:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695852.086482:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695852.086485:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695852.086488:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695852.086494:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695852.086496:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695852.086497:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695852.086500:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695852.086502:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695852.086504:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695852.086506:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695852.086508:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695852.086509:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695852.086598:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695852.086602:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695852.086605:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695852.470479:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695852.470486:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695852.470488:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695852.470493:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695852.470496:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695852.470499:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695852.470501:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695852.470503:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695852.470505:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695852.918467:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695852.918471:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695852.918479:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695852.918480:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695852.918483:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695852.918484:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695852.918489:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695852.918491:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695852.918494:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695852.918497:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695852.918503:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695852.918505:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695852.918507:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695852.918510:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695853.110488:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695853.110494:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695853.110500:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695853.110501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695853.110505:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695853.110507:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695853.110510:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695853.110511:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695853.110532:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695853.110534:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695853.110537:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695853.110538:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695853.110602:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695853.110605:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695853.110608:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695853.494471:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695853.494478:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695853.494479:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695853.494483:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695853.494486:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695853.494489:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695853.494491:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695853.494493:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695853.494495:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695853.942470:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695853.942470:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695853.942482:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695853.942483:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695853.942486:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695853.942488:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695853.942494:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695853.942494:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695853.942497:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695853.942498:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695853.942501:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695853.942503:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695853.942505:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695853.942508:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695854.134470:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695854.134478:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695854.134481:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695854.134485:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695854.134490:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695854.134492:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695854.134495:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695854.134497:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695854.134502:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695854.134503:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695854.134505:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695854.134507:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695854.134509:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695854.134510:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695854.134512:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695854.134514:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695854.134518:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695854.134520:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695854.134524:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695854.134526:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695854.134529:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695854.134530:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695854.134534:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695854.134535:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695854.134599:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695854.134604:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695854.134608:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695854.518417:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695854.518424:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695854.518443:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695854.518448:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695854.518451:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695854.518454:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695854.518456:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695854.518458:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695854.518460:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695854.966456:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695854.966463:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695854.966475:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695854.966476:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695854.966479:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695854.966481:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695854.966487:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695854.966489:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695854.966490:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695854.966495:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695854.966496:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695854.966507:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695854.966508:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695854.966523:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695855.158463:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695855.158471:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695855.158476:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695855.158477:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695855.158481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695855.158482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695855.158485:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695855.158486:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695855.158488:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695855.158489:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695855.158512:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695855.158513:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695855.158598:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695855.158604:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695855.158607:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695855.542431:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695855.542440:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695855.542441:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695855.542447:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695855.542450:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695855.542454:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695855.542456:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695855.542459:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695855.542462:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695855.990431:0:2405:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695855.990431:0:2406:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1787695855.990437:0:2405:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787695855.990437:0:2406:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000400:2.0:1787695855.990455:0:2405:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695770/real 1787695770] req@ffff8b14466f7b80 x1874531139520128/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695855 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:3.0:1787695855.990455:0:2406:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695770/real 1787695770] req@ffff8b14466f6a00 x1874531139520256/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695855 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695855.990486:0:2405:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695855.990489:0:2405:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695855.990491:0:2405:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695855.990495:0:2405:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695855.990497:0:2405:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695855.990501:0:2405:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000001:3.0:1787695855.990501:0:2406:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787695855.990505:0:2406:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000200:2.0:1787695855.990509:0:2405:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7b80 x1874531139520128/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695855 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1787695855.990510:0:2406:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14456d1688. 00000400:00000200:3.0:1787695855.990517:0:2406:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b145078a5d8 00000400:00000010:3.0:1787695855.990521:0:2406:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b145078a5d8. 00000100:00000200:2.0:1787695855.990524:0:2405:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7b80 x1874531139520128/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695855 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695855.990526:0:2406:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1787695855.990533:0:2406:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6a00 x1874531139520256/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695855 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1787695855.990540:0:2406:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6a00 x1874531139520256/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695855 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695855.990545:0:2405:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695855.990547:0:2405:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695855.990552:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695855.990561:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695855.990566:0:2406:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:2.0:1787695855.990570:0:2405:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b14466f7b80 x1874531139520128/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695855 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695855.990570:0:2406:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695855.990575:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695855.990578:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695855.990584:0:2405:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00100000:3.0:1787695855.990587:0:2406:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b14466f6a00 x1874531139520256/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695855 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695855.990588:0:2405:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b1458081800 already not connected (conn 29, was 15): DISCONN 00000100:00000001:2.0:1787695855.990594:0:2405:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:2.0:1787695855.990597:0:2405:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695855.990602:0:2406:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:3.0:1787695855.990604:0:2406:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b144515c000 already not connected (conn 25, was 11): DISCONN 00000100:00000001:2.0:1787695855.990606:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695855.990606:0:2406:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:2.0:1787695855.990610:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695855.990610:0:2406:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695855.990616:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695855.990618:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695855.990619:0:2405:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695855.990622:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695855.990623:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695855.990627:0:2405:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695855.990627:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695855.990630:0:2405:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695855.990630:0:2406:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695855.990634:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695855.990638:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7b80 x1874531139520128/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695855 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695855.990638:0:2406:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695855.990640:0:2406:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695855.990648:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6a00 x1874531139520256/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695855 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695855.990652:0:2405:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:2.0:1787695855.990658:0:2405:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695855.990662:0:2405:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695855.990663:0:2406:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:2.0:1787695855.990664:0:2405:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695855.990669:0:2406:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695855.990672:0:2405:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7b80 x1874531139520128/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695855 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695855.990673:0:2406:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695855.990676:0:2406:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695855.990683:0:2406:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6a00 x1874531139520256/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695855 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695855.990688:0:2405:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2405:1874531139520128:192.168.201.140@tcp:400:kworker.0 00000100:00000001:2.0:1787695855.990697:0:2405:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695855.990701:0:2405:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1787695855.990701:0:2406:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:608c526c-eb6e-4a54-80a5-8969828241eb:2406:1874531139520256:192.168.201.140@tcp:400:kworker.0 00000100:00000001:3.0:1787695855.990706:0:2406:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695855.990707:0:2405:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7b80 x1874531139520128/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695855 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695855.990710:0:2406:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695855.990716:0:2406:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6a00 x1874531139520256/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695855 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695855.990717:0:2405:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695855.990721:0:2405:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695855.990724:0:2405:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b14484ac400. 02000000:00000001:2.0:1787695855.990728:0:2405:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1787695855.990735:0:2406:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000020:00000001:2.0:1787695855.990737:0:2405:0:(genops.c:1217:class_import_put()) Process entered 02000000:00000001:3.0:1787695855.990738:0:2406:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000040:2.0:1787695855.990739:0:2405:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=9 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000010:3.0:1787695855.990741:0:2406:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1460530a00. 00000020:00000001:2.0:1787695855.990742:0:2405:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695855.990746:0:2405:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1458095600. 02000000:00000001:3.0:1787695855.990746:0:2406:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:2.0:1787695855.990752:0:2405:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695855.990753:0:2405:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695855.990755:0:2405:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7b80. 00000020:00000001:3.0:1787695855.990756:0:2406:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:2.0:1787695855.990758:0:2405:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000020:00000040:3.0:1787695855.990759:0:2406:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=10 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000001:2.0:1787695855.990760:0:2405:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695855.990764:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787695855.990765:0:2406:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:2.0:1787695855.990768:0:2405:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000010:3.0:1787695855.990768:0:2406:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1461888000. 00000100:00000001:2.0:1787695855.990771:0:2405:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695855.990773:0:2406:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1787695855.990774:0:2405:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787695855.990776:0:2406:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787695855.990777:0:2405:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695855.990779:0:2406:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6a00. 00000100:00000001:3.0:1787695855.990782:0:2406:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787695855.990784:0:2406:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695855.990785:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695855.990788:0:2406:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695855.990790:0:2406:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695855.990791:0:2406:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695855.990793:0:2406:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695856.182425:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695856.182431:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695856.182435:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695856.182436:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695856.182440:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695856.182442:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695856.182445:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695856.182449:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695856.182451:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695856.182452:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695856.182455:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695856.182457:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695856.182458:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695856.182460:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695856.182463:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695856.182464:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695856.182465:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695856.182466:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695856.182467:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695856.182468:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695856.182469:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695856.182469:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695856.182471:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695856.182472:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695856.182474:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695856.182475:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695856.182477:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695856.182478:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695856.182479:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695856.182480:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695856.182481:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695856.182482:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695856.182483:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695856.182483:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695856.182485:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695856.182486:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695856.182525:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695856.182529:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695856.182532:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787695856.502475:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695856.502485:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00080000:0.0:1787695856.502495:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787695856.502497:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787695856.502499:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1787695856.502501:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695856.502501:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.502505:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695856.502506:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695856.502507:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695856.502510:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695856.502514:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3744 00000100:00000001:1.0:1787695856.502514:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695856.502523:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695856.502529:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:0.0:1787695856.502533:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1787695856.502536:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695856.502540:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695856.502542:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695856.502545:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:0.0:1787695856.502549:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1787695856.502551:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:0.0:1787695856.502554:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1787695856.502558:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:0.0:1787695856.502560:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:0.0:1787695856.502563:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:0.0:1787695856.502566:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695856.502568:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695856.502570:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695856.502572:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:0.0:1787695856.502574:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:0.0:1787695856.502577:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695856.502580:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695856.502583:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787695856.502587:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695856.502590:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695856.502592:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787695856.502595:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1787695856.502601:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b145113f480. 00000020:00000040:0.0:1787695856.502610:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:0.0:1787695856.502614:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787695856.502616:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695856.502619:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695856.502621:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787695856.502626:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695856.502635:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1450775000. 00000100:00000001:0.0:1787695856.502641:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695856.502651:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b145113f480 x1874531139533952/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695856.502662:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695856.502667:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695856.502672:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000afccc75a] to pc [ptlrpcd_rcv+-1] req@ffff8b145113f480 x1874531139533952/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695856.502703:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695856.502708:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787695856.502719:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787695856.502721:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787695856.502723:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695856.502727:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787695856.502731:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3744 00000400:00000200:0.0:1787695856.502738:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695856.502740:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1787695856.502742:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695856.502745:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1787695856.502747:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695856.502750:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1787695856.502752:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695856.502756:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695856.502761:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b145113f480 x1874531139533952/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695856.502762:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695856.502766:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:0.0:1787695856.502772:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695856.502778:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695856.502780:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1787695856.502782:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:0.0:1787695856.502786:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1787695856.502786:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695856.502791:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695856.502797:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1787695856.502798:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:0.0:1787695856.502804:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00100000:1.0:1787695856.502806:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b145113f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139533952:192.168.201.140@tcp:250:kworker.0 00000020:00000001:0.0:1787695856.502808:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000001:1.0:1787695856.502816:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1787695856.502818:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695856.502822:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787695856.502823:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787695856.502828:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695856.502829:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695856.502834:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:1.0:1787695856.502835:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:0.0:1787695856.502838:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:0.0:1787695856.502843:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000010:1.0:1787695856.502844:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 00000100:00000001:0.0:1787695856.502848:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695856.502849:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695856.502852:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787695856.502858:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1787695856.502860:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:0.0:1787695856.502861:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695856.502865:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787695856.502869:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787695856.502871:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695856.502872:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695856.502874:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702c08. 00000100:00000010:0.0:1787695856.502876:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b145113d880. 00000020:00000040:0.0:1787695856.502880:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:0.0:1787695856.502882:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787695856.502883:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695856.502884:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695856.502886:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1787695856.502886:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 02000000:00000001:0.0:1787695856.502889:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695856.502894:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1450777000. 00000100:00000200:1.0:1787695856.502907:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139533952, portal 25 00000100:00000001:0.0:1787695856.502910:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.502912:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695856.502916:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615518108710016 : -128555600841600 : ffff8b145113f480) 00000100:00100000:0.0:1787695856.502918:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b145113d880 x1874531139534080/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1787695856.502928:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b145113f480 x1874531139533952/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695881 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695856.502929:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695856.502934:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695856.502942:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000036c280ff] to pc [ptlrpcd_rcv+-1] req@ffff8b145113d880 x1874531139534080/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.502948:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695856.502961:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695856.502966:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46e58. 00000100:00000200:1.0:1787695856.502972:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139533952, offset 0 00000100:00000001:0.0:1787695856.502976:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695856.502978:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787695856.502986:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787695856.502988:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787695856.502990:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1787695856.502993:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:0.0:1787695856.502996:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787695856.503002:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3744 00000400:00000200:1.0:1787695856.503003:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695856.503012:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695856.503016:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695856.503020:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695856.503023:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695856.503026:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695856.503028:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16981:16981:256:4294967295] 192.168.201.140@tcp LPNI seq info [16980:16980:8:4294967295] 00000400:00000200:1.0:1787695856.503032:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695856.503038:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695856.503045:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000400:00000200:0.0:1787695856.503047:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:0.0:1787695856.503053:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1787695856.503057:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000040:0.0:1787695856.503058:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695856.503062:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:1.0:1787695856.503063:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 00000100:00000040:0.0:1787695856.503065:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695856.503069:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:0.0:1787695856.503070:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1787695856.503071:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:0.0:1787695856.503074:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000200:1.0:1787695856.503075:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000020:00000040:0.0:1787695856.503077:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:0.0:1787695856.503081:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000800:00000200:1.0:1787695856.503081:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 13 secs later) 00000020:00000001:0.0:1787695856.503083:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:0.0:1787695856.503090:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000800:00000200:1.1:1787695856.503091:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695856.503094:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.1:1787695856.503097:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 13 secs later) 00000100:00000040:0.0:1787695856.503098:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695856.503103:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000100:1.0:1787695856.503107:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000020:00000040:0.0:1787695856.503108:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000800:00000010:1.0:1787695856.503112:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000100:00080000:0.0:1787695856.503114:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695856.503117:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695856.503118:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695856.503123:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 02000000:00000001:0.0:1787695856.503128:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1787695856.503130:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46e58 02000000:00000001:0.0:1787695856.503132:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1787695856.503134:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46e58. 02000000:00000001:0.0:1787695856.503137:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695856.503140:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695856.503140:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000008:00100000:0.0:1787695856.503144:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000200:1.0:1787695856.503146:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b145113f480 x1874531139533952/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695881 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1787695856.503147:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695856.503151:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1787695856.503156:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b145113df80. 00000020:00000040:0.0:1787695856.503160:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=10 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000100:00000001:1.0:1787695856.503163:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787695856.503164:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787695856.503166:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695856.503169:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b145113f480 x1874531139533952/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695881 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695856.503170:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695856.503173:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787695856.503175:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695856.503180:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1450774400. 00000100:00000001:1.0:1787695856.503182:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695856.503185:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.503185:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695856.503192:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695856.503201:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b145113df80 x1874531139534208/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695856.503202:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695856.503208:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298428216 00000100:00000001:0.0:1787695856.503211:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695856.503213:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695856.503217:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e69969e0] to pc [ptlrpcd_rcv+-1] req@ffff8b145113df80 x1874531139534208/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695856.503225:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695856.503227:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695856.503227:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298423097 00000100:00080000:0.0:1787695856.503233:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787695856.503235:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787695856.503236:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298428216 00000100:00080000:0.0:1787695856.503237:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695856.503242:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695856.503244:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298423097 00000100:00080000:0.0:1787695856.503250:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3744 00000100:00000001:1.0:1787695856.503250:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695856.503256:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695856.503261:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695856.503263:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000400:00000200:0.0:1787695856.503266:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:0.0:1787695856.503269:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787695856.503272:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695856/real 1787695856] req@ffff8b145113f480 x1874531139533952/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695856.503281:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695856.503286:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695856.503289:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000001:1.0:1787695856.503289:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1787695856.503291:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000001:1.0:1787695856.503292:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:0.0:1787695856.503294:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000010:1.0:1787695856.503296:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702c08. 00000020:00000040:0.0:1787695856.503297:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:0.0:1787695856.503301:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:1.0:1787695856.503302:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000020:00000040:0.0:1787695856.503303:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:0.0:1787695856.503307:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000400:00000010:1.0:1787695856.503307:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000020:00000001:0.0:1787695856.503313:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:1.0:1787695856.503314:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:0.0:1787695856.503317:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695856.503321:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1787695856.503322:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b145113f480 x1874531139533952/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695856.503325:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695856.503331:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:0.0:1787695856.503337:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00000200:1.0:1787695856.503339:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b145113f480 x1874531139533952/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787695856.503345:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695856.503349:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695856.503350:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.503352:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787695856.503356:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695856.503357:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695856.503361:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695856.503387:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b145113f480 x1874531139533952/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.503393:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695856.503398:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b145113f480 x1874531139533952/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787695856.503399:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695856.503406:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.503409:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695856.503411:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695856.503413:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:0.0:1787695856.503416:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787695856.503422:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.503423:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695856.503425:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695856.503428:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.503428:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695856.503432:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695856.503435:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000010:0.0:1787695856.503436:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b145113c380. 00000100:00080000:1.0:1787695856.503440:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000020:00000040:0.0:1787695856.503442:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000100:00000001:1.0:1787695856.503443:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:0.0:1787695856.503450:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000040:1.0:1787695856.503451:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b145113f480 x1874531139533952/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787695856.503453:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695856.503459:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695856.503462:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1787695856.503465:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b145113f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139533952:192.168.201.140@tcp:250:kworker.0 02000000:00000001:0.0:1787695856.503467:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.503470:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.503473:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000010:0.0:1787695856.503474:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1450775c00. 00000100:00000040:1.0:1787695856.503476:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b145113f480 x1874531139533952/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.503481:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695856.503483:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1787695856.503487:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695856.503487:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695856.503491:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695856.503495:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000100:00100000:0.0:1787695856.503496:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b145113c380 x1874531139534336/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000040:1.0:1787695856.503497:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:1.0:1787695856.503502:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695856.503505:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1450775000. 00000100:00000001:0.0:1787695856.503511:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787695856.503511:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1787695856.503515:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695856.503515:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695856.503517:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b145113f480. 00000100:00000040:0.0:1787695856.503521:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c87c79a1] to pc [ptlrpcd_rcv+-1] req@ffff8b145113c380 x1874531139534336/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.503522:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695856.503525:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.503529:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695856.503533:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695856.503535:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787695856.503541:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1787695856.503542:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787695856.503543:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1787695856.503544:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695856.503546:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695856.503548:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695856.503550:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1787695856.503552:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3744 00000100:00000001:1.0:1787695856.503553:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695856.503556:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000400:00000200:0.0:1787695856.503558:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695856.503561:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:0.0:1787695856.503565:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1787695856.503569:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695856.503570:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b145113d880 x1874531139534080/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695856.503572:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695856.503577:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695856.503581:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:0.0:1787695856.503584:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1787695856.503586:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:0.0:1787695856.503590:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695856.503590:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000040:0.0:1787695856.503592:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000100:00000001:1.0:1787695856.503594:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695856.503595:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:0.0:1787695856.503600:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 02000000:00000001:1.0:1787695856.503602:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1787695856.503603:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695856.503607:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787695856.503607:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695856.503612:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695856.503614:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695856.503617:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00100000:1.0:1787695856.503620:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b145113d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139534080:192.168.201.140@tcp:38:kworker.0 00000020:00000040:0.0:1787695856.503621:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:0.0:1787695856.503627:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695856.503631:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787695856.503632:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695856.503634:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787695856.503635:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695856.503639:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695856.503640:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695856.503642:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787695856.503645:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695856.503648:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000010:1.0:1787695856.503648:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 00000008:00100000:0.0:1787695856.503652:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 02000000:00000001:1.0:1787695856.503652:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787695856.503658:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695856.503662:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695856.503663:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695856.503666:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702c08. 00000100:00000010:0.0:1787695856.503669:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b145113ca80. 00000400:00000010:1.0:1787695856.503672:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000020:00000040:0.0:1787695856.503673:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:0.0:1787695856.503679:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000200:1.0:1787695856.503680:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139534080, portal 10 02000000:00000001:0.0:1787695856.503681:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.503685:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695856.503686:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787695856.503689:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615518108702848 : -128555600848768 : ffff8b145113d880) 02000000:00000001:0.0:1787695856.503690:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787695856.503693:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695856.503697:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1450775000. 00000100:00000040:1.0:1787695856.503699:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b145113d880 x1874531139534080/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695881 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695856.503701:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695856.503710:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b145113ca80 x1874531139534464/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695856.503711:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695856.503717:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695856.503722:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46e58. 00000100:00000001:0.0:1787695856.503724:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000200:1.0:1787695856.503725:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139534080, offset 0 00000100:00000001:0.0:1787695856.503728:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695856.503732:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:0.0:1787695856.503736:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f569ccc6] to pc [ptlrpcd_rcv+-1] req@ffff8b145113ca80 x1874531139534464/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695856.503741:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695856.503749:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:0.0:1787695856.503751:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695856.503754:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:0.0:1787695856.503755:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695856.503759:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00080000:0.0:1787695856.503760:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000400:00000200:1.0:1787695856.503763:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1787695856.503764:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787695856.503773:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1787695856.503776:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:0.0:1787695856.503778:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1787695856.503782:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16982:16982:256:4294967295] 192.168.201.140@tcp LPNI seq info [16981:16981:8:4294967295] 00000100:00080000:0.0:1787695856.503784:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3744 00000400:00000200:1.0:1787695856.503790:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695856.503795:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695856.503803:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000200:1.0:1787695856.503805:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00080000:0.0:1787695856.503807:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1787695856.503813:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695856.503817:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.0:1787695856.503818:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000040:0.0:1787695856.503822:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000010:1.0:1787695856.503824:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 00000100:00000001:0.0:1787695856.503825:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:0.0:1787695856.503829:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1787695856.503830:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000800:00000200:1.0:1787695856.503832:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000020:00000001:0.0:1787695856.503833:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1787695856.503837:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000800:00000200:1.0:1787695856.503838:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 13 secs later) 00000020:00000001:0.0:1787695856.503840:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000020:00000001:0.0:1787695856.503844:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000800:00000200:1.1:1787695856.503845:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:0.0:1787695856.503849:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000800:00000200:1.1:1787695856.503850:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 13 secs later) 00000100:00000001:0.0:1787695856.503853:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695856.503858:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000100:1.0:1787695856.503859:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00000001:0.0:1787695856.503860:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000010:1.0:1787695856.503863:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000020:00000040:0.0:1787695856.503864:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000400:00000200:1.0:1787695856.503868:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:0.0:1787695856.503869:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695856.503873:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695856.503876:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1787695856.503876:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 02000000:00000001:0.0:1787695856.503881:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1787695856.503882:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46e58 02000000:00000001:0.0:1787695856.503884:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695856.503886:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46e58. 00000100:00000001:0.0:1787695856.503887:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695856.503891:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695856.503894:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b145113d880 x1874531139534080/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695881 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787695856.503911:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787695856.503916:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695856.503919:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.503920:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695856.503924:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b145113d880 x1874531139534080/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695881 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1787695856.503928:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b145113e680. 00000020:00000040:0.0:1787695856.503933:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000100:00000001:1.0:1787695856.503934:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695856.503937:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787695856.503938:0:2402:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1787695856.503941:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.503943:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695856.503944:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695856.503947:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787695856.503949:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695856.503950:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 02000000:00000010:0.0:1787695856.503954:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b145101dc00. 00000400:00000200:1.0:1787695856.503955:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695856.503960:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.503960:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298418118 00000100:00000001:1.0:1787695856.503964:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298428216 00000100:00100000:0.0:1787695856.503969:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b145113e680 x1874531139534592/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695856.503969:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298418118 00000100:00000001:1.0:1787695856.503972:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298428216 00000100:00000001:0.0:1787695856.503977:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695856.503978:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.503984:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1787695856.503985:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000020113b06] to pc [ptlrpcd_rcv+-1] req@ffff8b145113e680 x1874531139534592/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.503987:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.503992:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787695856.503999:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695856.504000:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695856/real 1787695856] req@ffff8b145113d880 x1874531139534080/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787695856.504002:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787695856.504010:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787695856.504013:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787695856.504016:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:1.0:1787695856.504017:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1787695856.504018:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000001:1.0:1787695856.504019:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695856.504022:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702c08. 00000100:00080000:0.0:1787695856.504023:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3744 00000400:00000200:1.0:1787695856.504026:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:1.0:1787695856.504029:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000400:00000200:0.0:1787695856.504030:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695856.504032:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:0.0:1787695856.504035:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:0.0:1787695856.504040:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000200:1.0:1787695856.504040:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b145113d880 x1874531139534080/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695856.504047:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695856.504051:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695856.504055:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695856.504058:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000200:1.0:1787695856.504060:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b145113d880 x1874531139534080/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1787695856.504061:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695856.504066:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:0.0:1787695856.504069:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000100:00000001:1.0:1787695856.504070:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787695856.504073:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695856.504074:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:0.0:1787695856.504077:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000100:00000001:1.0:1787695856.504077:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787695856.504083:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00100000:1.0:1787695856.504085:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b145113d880 x1874531139534080/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1787695856.504086:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:0.0:1787695856.504094:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695856.504098:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695856.504102:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695856.504103:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695856.504107:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000040:1.0:1787695856.504111:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b145113d880 x1874531139534080/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1787695856.504112:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00080000:0.0:1787695856.504118:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695856.504123:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695856.504127:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695856.504129:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1787695856.504132:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695856.504133:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787695856.504136:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.504137:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695856.504140:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695856.504141:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:0.0:1787695856.504144:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787695856.504144:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695856.504148:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:0.0:1787695856.504149:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695856.504150:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1787695856.504152:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b145113c700. 00000020:00000040:0.0:1787695856.504154:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=11 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:0.0:1787695856.504159:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00080000:1.0:1787695856.504160:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 02000000:00000001:0.0:1787695856.504161:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695856.504164:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695856.504166:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00080000:1.0:1787695856.504168:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 02000000:00000001:0.0:1787695856.504169:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.504171:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000010:0.0:1787695856.504175:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f4400. 00000100:00000040:1.0:1787695856.504176:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b145113d880 x1874531139534080/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695856.504179:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695856.504188:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b145113c700 x1874531139534720/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787695856.504191:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b145113d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139534080:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695856.504200:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787695856.504201:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695856.504204:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695856.504208:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b145113df80 x1874531139534208/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695856.504216:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ad28be0a] to pc [ptlrpcd_rcv+-1] req@ffff8b145113c700 x1874531139534720/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695856.504227:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695856.504232:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695856.504235:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1787695856.504236:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1787695856.504238:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695856.504240:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1787695856.504245:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 02000000:00000001:1.0:1787695856.504245:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1787695856.504248:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3754) 02000000:00000001:1.0:1787695856.504250:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695856.504253:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695856.504257:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b145113df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139534208:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695856.504260:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695856.504263:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695856.504264:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695856.504266:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695856.504269:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695856.504271:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695856.504274:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695856.504276:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702c08. 00000400:00000010:1.0:1787695856.504278:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000100:00000200:1.0:1787695856.504281:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139534208, portal 10 00000100:00000001:1.0:1787695856.504282:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695856.504283:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615518108704640 : -128555600846976 : ffff8b145113df80) 00000100:00000040:1.0:1787695856.504288:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b145113df80 x1874531139534208/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695881 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504301:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695856.504303:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695856.504304:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46e58. 00000100:00000200:1.0:1787695856.504307:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139534208, offset 0 00000400:00000200:1.0:1787695856.504309:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695856.504312:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695856.504315:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695856.504317:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695856.504319:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695856.504321:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695856.504322:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695856.504325:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16983:16983:256:4294967295] 192.168.201.140@tcp LPNI seq info [16982:16982:8:4294967295] 00000400:00000200:1.0:1787695856.504327:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695856.504332:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695856.504335:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695856.504337:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 00000800:00000200:1.0:1787695856.504340:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695856.504342:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 13 secs later) 00000800:00000200:1.1:1787695856.504344:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695856.504346:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 13 secs later) 00000800:00000100:1.0:1787695856.504348:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695856.504350:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000400:00000200:1.0:1787695856.504352:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695856.504355:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695856.504357:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46e58 00000400:00000010:1.0:1787695856.504358:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46e58. 00000100:00000001:1.0:1787695856.504360:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695856.504378:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b145113df80 x1874531139534208/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695881 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504385:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695856.504387:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b145113df80 x1874531139534208/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695881 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504401:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.504402:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695856.504405:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695856.504407:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787695856.504410:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695856.504411:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298416967 00000100:00000001:1.0:1787695856.504413:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298428216 00000100:00000001:1.0:1787695856.504415:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298416967 00000100:00000001:1.0:1787695856.504416:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298428216 00000100:00000001:1.0:1787695856.504417:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695856.504419:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.504420:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695856.504423:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695856/real 1787695856] req@ffff8b145113df80 x1874531139534208/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695856.504427:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695856.504428:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695856.504429:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702c08. 00000400:00000200:1.0:1787695856.504431:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:1.0:1787695856.504432:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000100:00000001:1.0:1787695856.504433:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695856.504435:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b145113df80 x1874531139534208/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695856.504440:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b145113df80 x1874531139534208/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504445:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695856.504446:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.504447:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695856.504448:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695856.504450:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b145113df80 x1874531139534208/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504457:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695856.504460:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b145113df80 x1874531139534208/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504465:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695856.504466:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.504467:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695856.504468:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.504469:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695856.504470:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695856.504472:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695856.504474:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695856.504475:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695856.504477:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b145113df80 x1874531139534208/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695856.504482:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b145113df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139534208:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695856.504485:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695856.504487:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b145113c380 x1874531139534336/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504491:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695856.504492:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695856.504493:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695856.504495:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695856.504496:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695856.504498:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b145113c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139534336:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695856.504499:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695856.504501:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695856.504506:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695856.504507:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695856.504509:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695856.504510:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695856.504513:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695856.504515:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702c08. 00000400:00000010:1.0:1787695856.504517:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000100:00000200:1.0:1787695856.504519:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139534336, portal 4 00000100:00000001:1.0:1787695856.504521:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695856.504522:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615518108697472 : -128555600854144 : ffff8b145113c380) 00000100:00000040:1.0:1787695856.504526:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b145113c380 x1874531139534336/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695881 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504530:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695856.504531:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695856.504533:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46e58. 00000100:00000200:1.0:1787695856.504534:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139534336, offset 0 00000400:00000200:1.0:1787695856.504537:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695856.504539:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695856.504541:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695856.504543:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695856.504545:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695856.504546:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695856.504548:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695856.504550:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16984:16984:256:4294967295] 192.168.201.140@tcp LPNI seq info [16983:16983:8:4294967295] 00000400:00000200:1.0:1787695856.504553:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695856.504557:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695856.504560:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695856.504562:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 00000800:00000200:1.0:1787695856.504565:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695856.504567:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 13 secs later) 00000800:00000200:1.1:1787695856.504570:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695856.504571:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 13 secs later) 00000800:00000100:1.0:1787695856.504574:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695856.504580:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000400:00000200:1.0:1787695856.504583:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695856.504587:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695856.504591:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46e58 00000400:00000010:1.0:1787695856.504593:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46e58. 00000100:00000001:1.0:1787695856.504596:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695856.504600:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b145113c380 x1874531139534336/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695881 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504609:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695856.504612:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b145113c380 x1874531139534336/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695881 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504619:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.504621:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695856.504625:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695856.504628:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695856.504630:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695856.504632:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298416997 00000100:00000001:1.0:1787695856.504634:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298428216 00000100:00000001:1.0:1787695856.504635:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298416997 00000100:00000001:1.0:1787695856.504637:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298428216 00000100:00000001:1.0:1787695856.504638:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695856.504640:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.504641:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695856.504644:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695856/real 1787695856] req@ffff8b145113c380 x1874531139534336/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695856.504648:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695856.504649:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695856.504650:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702c08. 00000400:00000200:1.0:1787695856.504652:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:1.0:1787695856.504653:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000100:00000001:1.0:1787695856.504659:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695856.504662:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b145113c380 x1874531139534336/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695856.504667:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b145113c380 x1874531139534336/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504672:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695856.504673:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.504674:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695856.504675:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695856.504677:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b145113c380 x1874531139534336/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504681:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695856.504683:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b145113c380 x1874531139534336/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504688:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695856.504689:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.504690:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695856.504691:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.504692:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695856.504693:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695856.504695:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695856.504697:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695856.504698:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695856.504700:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b145113c380 x1874531139534336/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695856.504705:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b145113c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139534336:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695856.504707:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.504708:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695856.504714:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b145113d880 x1874531139534080/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504718:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695856.504720:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695856.504721:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695856.504723:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695856.504724:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695856.504725:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695856.504726:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695856.504728:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1450777000. 02000000:00000001:1.0:1787695856.504730:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695856.504731:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695856.504732:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b145113d880. 00000100:00000001:1.0:1787695856.504734:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695856.504735:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.504736:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695856.504738:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b145113df80 x1874531139534208/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504742:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695856.504743:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695856.504744:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695856.504746:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695856.504747:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695856.504748:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=9 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695856.504749:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695856.504749:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1450774400. 02000000:00000001:1.0:1787695856.504751:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695856.504752:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695856.504753:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b145113df80. 00000100:00000001:1.0:1787695856.504754:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695856.504755:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.504756:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695856.504758:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b145113c380 x1874531139534336/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504762:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695856.504765:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695856.504766:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695856.504770:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695856.504771:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695856.504772:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695856.504773:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695856.504774:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1450775c00. 02000000:00000001:1.0:1787695856.504775:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695856.504776:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695856.504776:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b145113c380. 00000100:00000001:1.0:1787695856.504778:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695856.504779:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.504780:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.504782:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695856.504783:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.504785:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695856.504785:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695856.504786:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695856.504789:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b145113ca80 x1874531139534464/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504793:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695856.504794:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695856.504795:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695856.504796:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695856.504797:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695856.504799:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b145113ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139534464:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695856.504802:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695856.504803:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695856.504804:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695856.504805:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695856.504807:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695856.504808:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695856.504810:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695856.504812:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702c08. 00000400:00000010:1.0:1787695856.504814:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000100:00000200:1.0:1787695856.504816:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139534464, portal 4 00000100:00000001:1.0:1787695856.504820:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695856.504821:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615518108699264 : -128555600852352 : ffff8b145113ca80) 00000100:00000040:1.0:1787695856.504825:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b145113ca80 x1874531139534464/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695881 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504830:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695856.504831:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695856.504833:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46e58. 00000100:00000200:1.0:1787695856.504835:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139534464, offset 0 00000400:00000200:1.0:1787695856.504837:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695856.504840:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695856.504842:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695856.504844:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695856.504846:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695856.504848:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695856.504849:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695856.504851:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16985:16985:256:4294967295] 192.168.201.140@tcp LPNI seq info [16984:16984:8:4294967295] 00000400:00000200:1.0:1787695856.504854:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695856.504858:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695856.504861:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695856.504863:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 00000800:00000200:1.0:1787695856.504866:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695856.504868:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 13 secs later) 00000800:00000200:1.1:1787695856.504870:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695856.504872:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 13 secs later) 00000800:00000100:1.0:1787695856.504875:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695856.504876:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000400:00000200:1.0:1787695856.504878:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695856.504881:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695856.504882:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46e58 00000400:00000010:1.0:1787695856.504883:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46e58. 00000100:00000001:1.0:1787695856.504885:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695856.504887:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b145113ca80 x1874531139534464/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695881 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504916:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695856.504919:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b145113ca80 x1874531139534464/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695881 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504923:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.504925:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695856.504928:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695856.504930:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695856.504932:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695856.504934:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298416669 00000100:00000001:1.0:1787695856.504936:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298428216 00000100:00000001:1.0:1787695856.504937:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298416669 00000100:00000001:1.0:1787695856.504939:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298428216 00000100:00000001:1.0:1787695856.504940:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695856.504942:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.504943:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695856.504945:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695856/real 1787695856] req@ffff8b145113ca80 x1874531139534464/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695856.504950:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695856.504951:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695856.504952:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702c08. 00000400:00000200:1.0:1787695856.504954:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:1.0:1787695856.504955:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000100:00000001:1.0:1787695856.504957:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695856.504959:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b145113ca80 x1874531139534464/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695856.504964:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b145113ca80 x1874531139534464/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504968:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695856.504973:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.504975:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695856.504976:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695856.504978:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b145113ca80 x1874531139534464/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504983:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695856.504985:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b145113ca80 x1874531139534464/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.504990:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695856.504991:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.504992:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695856.504993:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.504994:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695856.504995:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695856.504996:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695856.504998:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695856.505000:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695856.505002:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b145113ca80 x1874531139534464/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695856.505007:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b145113ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139534464:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695856.505009:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695856.505011:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b145113e680 x1874531139534592/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.505015:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695856.505016:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695856.505017:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695856.505019:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695856.505020:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695856.505025:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b145113e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139534592:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695856.505027:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695856.505028:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695856.505029:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695856.505031:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695856.505032:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695856.505033:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695856.505036:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695856.505037:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702c08. 00000400:00000010:1.0:1787695856.505039:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000100:00000200:1.0:1787695856.505041:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139534592, portal 10 00000100:00000001:1.0:1787695856.505042:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695856.505044:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615518108706432 : -128555600845184 : ffff8b145113e680) 00000100:00000040:1.0:1787695856.505048:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b145113e680 x1874531139534592/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695881 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.505052:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695856.505054:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695856.505055:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46e58. 00000100:00000200:1.0:1787695856.505056:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139534592, offset 0 00000400:00000200:1.0:1787695856.505059:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695856.505061:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695856.505063:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695856.505065:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695856.505067:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695856.505069:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695856.505070:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695856.505072:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16986:16986:256:4294967295] 192.168.201.140@tcp LPNI seq info [16985:16985:8:4294967295] 00000400:00000200:1.0:1787695856.505075:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695856.505079:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695856.505083:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695856.505085:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 00000800:00000200:1.0:1787695856.505091:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695856.505092:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 13 secs later) 00000800:00000200:1.1:1787695856.505095:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695856.505097:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 13 secs later) 00000800:00000100:1.0:1787695856.505099:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695856.505101:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000400:00000200:1.0:1787695856.505103:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695856.505106:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695856.505108:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46e58 00000400:00000010:1.0:1787695856.505109:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46e58. 00000100:00000001:1.0:1787695856.505110:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695856.505113:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b145113e680 x1874531139534592/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695881 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.505118:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695856.505120:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b145113e680 x1874531139534592/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695881 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.505124:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.505126:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695856.505127:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695856.505129:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695856.505131:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695856.505132:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298416855 00000100:00000001:1.0:1787695856.505134:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298428216 00000100:00000001:1.0:1787695856.505135:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298416855 00000100:00000001:1.0:1787695856.505137:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298428216 00000100:00000001:1.0:1787695856.505138:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695856.505140:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.505141:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695856.505143:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695856/real 1787695856] req@ffff8b145113e680 x1874531139534592/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695856.505147:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695856.505151:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695856.505153:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702c08. 00000400:00000200:1.0:1787695856.505154:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:1.0:1787695856.505155:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000100:00000001:1.0:1787695856.505157:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695856.505159:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b145113e680 x1874531139534592/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695856.505165:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b145113e680 x1874531139534592/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.505169:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695856.505170:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.505171:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695856.505172:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695856.505174:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b145113e680 x1874531139534592/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.505178:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695856.505180:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b145113e680 x1874531139534592/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.505184:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695856.505186:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.505187:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695856.505188:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.505189:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695856.505191:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695856.505193:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695856.505196:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695856.505199:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695856.505203:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b145113e680 x1874531139534592/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695856.505222:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b145113e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139534592:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695856.505226:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695856.505230:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b145113c700 x1874531139534720/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.505237:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695856.505239:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695856.505241:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695856.505243:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695856.505245:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695856.505247:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b145113c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139534720:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695856.505250:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695856.505251:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695856.505252:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695856.505253:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695856.505256:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695856.505257:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695856.505259:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695856.505261:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702c08. 00000400:00000010:1.0:1787695856.505264:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000100:00000200:1.0:1787695856.505266:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139534720, portal 10 00000100:00000001:1.0:1787695856.505267:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695856.505268:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615518108698368 : -128555600853248 : ffff8b145113c700) 00000100:00000040:1.0:1787695856.505272:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b145113c700 x1874531139534720/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695881 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.505277:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695856.505278:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695856.505280:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46e58. 00000100:00000200:1.0:1787695856.505281:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139534720, offset 0 00000400:00000200:1.0:1787695856.505284:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695856.505290:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695856.505293:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695856.505295:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695856.505296:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695856.505298:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695856.505300:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695856.505302:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16987:16987:256:4294967295] 192.168.201.140@tcp LPNI seq info [16986:16986:8:4294967295] 00000400:00000200:1.0:1787695856.505305:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695856.505309:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695856.505312:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695856.505314:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 00000800:00000200:1.0:1787695856.505316:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695856.505318:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 13 secs later) 00000800:00000200:1.1:1787695856.505321:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695856.505323:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 13 secs later) 00000800:00000100:1.0:1787695856.505325:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695856.505326:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000400:00000200:1.0:1787695856.505329:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695856.505331:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695856.505333:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46e58 00000400:00000010:1.0:1787695856.505334:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46e58. 00000100:00000001:1.0:1787695856.505336:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695856.505338:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b145113c700 x1874531139534720/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695881 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.505343:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695856.505345:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b145113c700 x1874531139534720/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695881 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.505350:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.505351:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695856.505353:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695856.505355:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695856.505357:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695856.505362:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298416833 00000100:00000001:1.0:1787695856.505379:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298428216 00000100:00000001:1.0:1787695856.505382:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298416834 00000100:00000001:1.0:1787695856.505383:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298428216 00000100:00000001:1.0:1787695856.505386:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695856.505388:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.505389:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695856.505393:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695856/real 1787695856] req@ffff8b145113c700 x1874531139534720/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695856.505399:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695856.505401:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695856.505402:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702c08. 00000400:00000200:1.0:1787695856.505405:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:1.0:1787695856.505406:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000100:00000001:1.0:1787695856.505408:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695856.505411:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b145113c700 x1874531139534720/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695856.505417:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b145113c700 x1874531139534720/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.505422:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695856.505424:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.505425:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695856.505426:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695856.505429:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b145113c700 x1874531139534720/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.505433:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695856.505435:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b145113c700 x1874531139534720/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.505439:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695856.505445:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.505446:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695856.505447:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.505448:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695856.505449:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695856.505451:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695856.505453:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695856.505454:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695856.505457:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b145113c700 x1874531139534720/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695856.505462:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b145113c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139534720:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695856.505464:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.505466:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695856.505468:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b145113ca80 x1874531139534464/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.505471:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695856.505472:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695856.505474:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695856.505476:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695856.505477:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695856.505478:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695856.505479:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695856.505480:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1450775000. 02000000:00000001:1.0:1787695856.505482:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695856.505483:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695856.505484:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b145113ca80. 00000100:00000001:1.0:1787695856.505485:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695856.505486:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.505487:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695856.505489:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b145113e680 x1874531139534592/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.505500:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695856.505500:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695856.505502:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695856.505503:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695856.505504:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695856.505505:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695856.505506:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695856.505507:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b145101dc00. 02000000:00000001:1.0:1787695856.505508:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695856.505509:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695856.505510:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b145113e680. 00000100:00000001:1.0:1787695856.505511:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695856.505512:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.505513:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695856.505515:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b145113c700 x1874531139534720/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695881 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695856.505519:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695856.505519:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695856.505520:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695856.505522:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695856.505522:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695856.505523:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=10 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695856.505524:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695856.505525:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f4400. 02000000:00000001:1.0:1787695856.505527:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695856.505528:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695856.505528:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b145113c700. 00000100:00000001:1.0:1787695856.505530:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695856.505530:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.505532:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695856.505534:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695856.505535:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695856.505536:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695856.505537:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695856.566473:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695856.566483:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695856.566485:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695856.566490:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695856.566493:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695856.566497:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695856.566498:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695856.566501:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695856.566503:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695857.014455:0:2403:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695857.014471:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695857.014473:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695857.014478:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695857.014480:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695857.014484:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695857.014488:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695857.206487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695857.206518:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695857.206524:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695857.206525:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695857.206528:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695857.206529:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695857.206532:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695857.206533:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695857.206535:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695857.206536:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695857.206538:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695857.206539:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695857.206593:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695857.206597:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695857.206600:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787695857.526444:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695857.526456:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695857.526458:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695857.526462:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695857.526464:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695857.526468:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695857.526470:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695857.590463:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695857.590469:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695857.590471:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695857.590475:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695857.590478:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695857.590482:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695857.590499:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695857.590502:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695857.590504:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695858.038444:0:2403:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695858.038455:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695858.038456:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695858.038460:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695858.038462:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695858.038466:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695858.038469:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695858.230459:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695858.230467:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695858.230474:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695858.230475:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695858.230479:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695858.230480:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695858.230484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695858.230486:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695858.230489:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695858.230491:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695858.230495:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695858.230496:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695858.230550:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695858.230554:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695858.230557:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787695858.550424:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695858.550438:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695858.550440:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695858.550446:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695858.550449:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695858.550453:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695858.550456:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695858.614473:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695858.614482:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695858.614484:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695858.614491:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695858.614495:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695858.614499:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695858.614501:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695858.614505:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695858.614508:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695859.062462:0:2403:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787695859.062476:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695859.062479:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695859.062484:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695859.062486:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695859.062490:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695859.062495:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695859.254475:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695859.254481:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695859.254501:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695859.254502:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695859.254505:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695859.254506:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695859.254509:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695859.254509:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695859.254512:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695859.254513:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695859.254516:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695859.254518:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695859.254576:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695859.254580:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695859.254589:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787695859.574684:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695859.574697:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695859.574699:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695859.574704:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695859.574707:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695859.574711:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695859.574715:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695859.638447:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695859.638454:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695859.638457:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695859.638463:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695859.638467:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695859.638478:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695859.638481:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695859.638484:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695859.638487:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695860.086498:0:2403:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695860.086513:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695860.086515:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695860.086521:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695860.086523:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695860.086527:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695860.086530:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695860.278438:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695860.278446:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695860.278452:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695860.278453:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695860.278456:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695860.278457:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695860.278460:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695860.278461:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695860.278464:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695860.278465:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695860.278467:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695860.278468:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695860.278519:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695860.278522:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695860.278525:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787695860.598527:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695860.598543:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695860.598546:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695860.598562:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695860.598564:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695860.598568:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695860.598572:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695860.662461:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695860.662469:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695860.662471:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695860.662477:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695860.662481:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695860.662485:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695860.662488:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695860.662490:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695860.662493:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.110493:0:2403:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695861.110502:0:2403:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000400:1.0:1787695861.110517:0:2403:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695776/real 1787695776] req@ffff8b1446665500 x1874531139520512/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695861 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695861.110536:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695861.110538:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695861.110542:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b630. 00000400:00000200:1.0:1787695861.110550:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f96aa0 00000400:00000010:1.0:1787695861.110553:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f96aa0. 00000100:00000001:1.0:1787695861.110557:0:2403:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695861.110561:0:2403:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665500 x1874531139520512/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695861 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695861.110572:0:2403:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665500 x1874531139520512/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695861 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.110581:0:2403:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695861.110583:0:2403:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.110586:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695861.110589:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695861.110594:0:2403:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446665500 x1874531139520512/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695861 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.110603:0:2403:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:1.0:1787695861.110606:0:2403:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b1458081800 already not connected (conn 30, was 15): DISCONN 00000100:00000001:1.0:1787695861.110609:0:2403:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:1.0:1787695861.110611:0:2403:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.110620:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695861.110622:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.110626:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695861.110628:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695861.110631:0:2403:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.110651:0:2403:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695861.110654:0:2403:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695861.110660:0:2403:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665500 x1874531139520512/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695861 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.110669:0:2403:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:1.0:1787695861.110672:0:2403:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.110674:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695861.110675:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695861.110680:0:2403:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665500 x1874531139520512/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695861 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695861.110689:0:2403:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2403:1874531139520512:192.168.201.140@tcp:400:kworker.0 00000100:00000001:1.0:1787695861.110695:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.110698:0:2403:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695861.110702:0:2403:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665500 x1874531139520512/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695861 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.110709:0:2403:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695861.110713:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695861.110717:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1447625a00. 02000000:00000001:1.0:1787695861.110722:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695861.110725:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695861.110727:0:2403:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695861.110730:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695861.110732:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1445678f00. 02000000:00000001:1.0:1787695861.110737:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695861.110738:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695861.110740:0:2403:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665500. 00000100:00000001:1.0:1787695861.110744:0:2403:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695861.110746:0:2403:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.110749:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695861.302441:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695861.302446:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695861.302449:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695861.302464:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695861.302467:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695861.302468:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695861.302471:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695861.302472:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695861.302474:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695861.302477:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695861.302479:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695861.302484:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695861.302485:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695861.302487:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695861.302489:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695861.302491:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695861.302492:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695861.302493:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695861.302496:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695861.302497:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695861.302547:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695861.302553:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695861.302556:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787695861.622537:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695861.622546:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787695861.622548:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695861.622551:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695861.622570:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695861.622583:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3749 00000400:00000200:3.0:1787695861.622596:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695861.622604:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695861.622607:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695861.622610:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695861.622614:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695861.622618:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695861.622620:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695861.622629:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695861.622631:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:3.0:1787695861.622634:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695861.622637:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787695861.622640:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787695861.622642:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787695861.622646:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695861.622648:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695861.622650:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695861.622652:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695861.622656:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787695861.622659:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695861.622662:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695861.622665:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695861.622669:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695861.622672:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695861.622673:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787695861.622675:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695861.622681:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6a00. 00000020:00000040:3.0:1787695861.622684:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787695861.622687:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695861.622688:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695861.622698:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695861.622700:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695861.622704:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695861.622709:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675800. 00000100:00000001:3.0:1787695861.622715:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695861.622723:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6a00 x1874531139534848/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695861.622731:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695861.622734:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695861.622738:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002ad47e8f] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6a00 x1874531139534848/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695861.622757:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695861.622759:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695861.622762:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695861.622763:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695861.622764:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695861.622767:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695861.622770:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3749 00000400:00000200:3.0:1787695861.622774:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695861.622779:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695861.622782:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695861.622784:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695861.622786:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695861.622787:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695861.622788:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695861.622790:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695861.622791:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:3.0:1787695861.622794:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695861.622796:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:3.0:1787695861.622798:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:3.0:1787695861.622801:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:3.0:1787695861.622823:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695861.622826:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695861.622828:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695861.622830:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695861.622832:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00000001:1.0:1787695861.622835:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787695861.622837:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695861.622845:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.622847:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787695861.622847:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787695861.622852:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695861.622856:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787695861.622859:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695861.622860:0:2404:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1787695861.622867:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695861.622873:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6a00 x1874531139534848/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695861.622873:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000400:0.0:1787695861.622875:0:2404:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695776/real 1787695776] req@ffff8b1446667100 x1874531139520640/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695861 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787695861.622883:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695861.622900:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695861.622911:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695861.622922:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4a80. 00000020:00000040:3.0:1787695861.622931:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695861.622933:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695861.622934:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695861.622935:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695861.622936:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695861.622938:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695861.622948:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 00000100:00000001:1.0:1787695861.622950:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695861.622959:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695861.622960:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787695861.622965:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787695861.622973:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:1.0:1787695861.622977:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:3.0:1787695861.622979:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4a80 x1874531139534976/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:0.0:1787695861.622982:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702b00. 02000000:00000001:1.0:1787695861.622983:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1787695861.622996:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46770 02000000:00000001:1.0:1787695861.622998:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1787695861.623005:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46770. 00000100:00000001:0.0:1787695861.623017:0:2404:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787695861.623018:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1787695861.623019:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139534848:192.168.201.140@tcp:250:kworker.0 00000100:00000001:3.0:1787695861.623026:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695861.623028:0:2404:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446667100 x1874531139520640/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695861 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.623038:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1787695861.623044:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ad0d67] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4a80 x1874531139534976/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695861.623050:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695861.623059:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1787695861.623072:0:2404:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446667100 x1874531139520640/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695861 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695861.623074:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787695861.623074:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695861.623085:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787695861.623090:0:2404:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000010:1.0:1787695861.623094:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 00000100:00000001:0.0:1787695861.623095:0:2404:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695861.623098:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787695861.623107:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1787695861.623107:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695861.623107:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787695861.623121:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695861.623122:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1787695861.623134:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695861.623137:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:0.0:1787695861.623139:0:2404:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446667100 x1874531139520640/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695861 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1787695861.623141:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702c08. 00000100:00080000:3.0:1787695861.623145:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3749 00000400:00000010:1.0:1787695861.623163:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000400:00000200:3.0:1787695861.623163:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695861.623166:0:2404:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:0.0:1787695861.623178:0:2404:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b144515c000 already not connected (conn 26, was 11): DISCONN 00000400:00000200:3.0:1787695861.623180:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000200:1.0:1787695861.623183:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139534848, portal 25 00000100:00080000:3.0:1787695861.623194:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787695861.623195:0:2404:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:1.0:1787695861.623203:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695861.623204:0:2404:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.623209:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930154496 : -128555779397120 : ffff8b14466f6a00) 00000100:00000040:3.0:1787695861.623210:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695861.623217:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787695861.623220:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787695861.623223:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695861.623227:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6a00 x1874531139534848/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695886 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695861.623228:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695861.623234:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1787695861.623234:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695861.623241:0:2404:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695861.623244:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695861.623254:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695861.623257:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:3.0:1787695861.623266:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000100:00000001:0.0:1787695861.623268:0:2404:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1787695861.623275:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000020:00000001:3.0:1787695861.623276:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787695861.623277:0:2404:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695861.623286:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000400:00000010:1.0:1787695861.623288:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46e58. 00000020:00000001:3.0:1787695861.623290:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000200:1.0:1787695861.623293:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139534848, offset 0 00000100:00000040:0.0:1787695861.623298:0:2404:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446667100 x1874531139520640/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695861 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695861.623302:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000400:00000200:1.0:1787695861.623314:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695861.623316:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695861.623319:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695861.623330:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695861.623333:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695861.623335:0:2404:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:3.0:1787695861.623343:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:0.0:1787695861.623346:0:2404:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1787695861.623347:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000400:00000200:1.0:1787695861.623353:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:0.0:1787695861.623355:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695861.623361:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1787695861.623401:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:0.0:1787695861.623406:0:2404:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446667100 x1874531139520640/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695861 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695861.623410:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:1.0:1787695861.623411:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:3.0:1787695861.623421:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695861.623425:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 02000000:00000001:3.0:1787695861.623433:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1787695861.623437:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 02000000:00000001:3.0:1787695861.623442:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1787695861.623450:0:2404:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446667100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:608c526c-eb6e-4a54-80a5-8969828241eb:2404:1874531139520640:192.168.201.140@tcp:400:kworker.0 00000400:00000200:1.0:1787695861.623453:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16988:16988:256:4294967295] 192.168.201.140@tcp LPNI seq info [16987:16987:8:4294967295] 02000000:00000001:3.0:1787695861.623454:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695861.623461:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695861.623467:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787695861.623470:0:2404:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000008:00100000:3.0:1787695861.623472:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000040:0.0:1787695861.623477:0:2404:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446667100 x1874531139520640/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695861 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695861.623477:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000008:00000001:3.0:1787695861.623491:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695861.623502:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695861.623517:0:2404:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000400:00000200:1.0:1787695861.623517:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000010:3.0:1787695861.623519:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4000. 02000000:00000001:0.0:1787695861.623523:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000040:3.0:1787695861.623534:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=9 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000010:0.0:1787695861.623535:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1447533200. 00000800:00000200:1.0:1787695861.623536:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 02000000:00000001:0.0:1787695861.623550:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:3.0:1787695861.623551:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000010:1.0:1787695861.623552:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 02000000:00000001:3.0:1787695861.623561:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695861.623562:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:0.0:1787695861.623568:0:2404:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000001:3.0:1787695861.623570:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.0:1787695861.623572:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000020:00000001:0.0:1787695861.623577:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787695861.623578:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:1.0:1787695861.623580:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 8 secs later) 02000000:00000010:0.0:1787695861.623583:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1445678900. 02000000:00000001:3.0:1787695861.623585:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695861.623589:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000800:00000200:1.1:1787695861.623593:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:0.0:1787695861.623594:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000010:3.0:1787695861.623598:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 00000100:00000010:0.0:1787695861.623599:0:2404:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446667100. 00000800:00000200:1.1:1787695861.623599:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 8 secs later) 00000100:00000001:0.0:1787695861.623611:0:2404:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000800:00000100:1.0:1787695861.623615:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00000001:0.0:1787695861.623617:0:2404:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695861.623618:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1787695861.623619:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000400:00000200:1.0:1787695861.623630:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1787695861.623637:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695861.623639:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4000 x1874531139535104/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695861.623645:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695861.623662:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46e58 00000100:00000001:3.0:1787695861.623662:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695861.623668:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695861.623670:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46e58. 00000100:00000040:3.0:1787695861.623675:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5a8f356] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4000 x1874531139535104/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.623679:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787695861.623683:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695861.623686:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695861.623691:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787695861.623693:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695861.623696:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1787695861.623701:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6a00 x1874531139534848/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695886 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695861.623701:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695861.623708:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3749 00000400:00000200:3.0:1787695861.623721:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695861.623734:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695861.623743:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695861.623744:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695861.623748:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695861.623757:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6a00 x1874531139534848/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695886 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695861.623758:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695861.623770:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695861.623775:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.623779:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787695861.623785:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695861.623789:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695861.623790:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000100:00000001:1.0:1787695861.623793:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695861.623793:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695861.623799:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000400:00000200:1.0:1787695861.623807:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:3.0:1787695861.623821:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:1.0:1787695861.623826:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298433337 00000020:00000001:3.0:1787695861.623826:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:3.0:1787695861.623839:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695861.623841:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695861.623842:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298428216 00000100:00000040:3.0:1787695861.623843:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695861.623850:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695861.623853:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298433337 00000020:00000040:3.0:1787695861.623854:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00000001:1.0:1787695861.623870:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298428216 00000100:00080000:3.0:1787695861.623871:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695861.623882:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.623885:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695861.623886:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695861.623896:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695861.623897:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695861.623905:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1787695861.623905:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695861.623913:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695861.623915:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787695861.623921:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695861/real 1787695861] req@ffff8b14466f6a00 x1874531139534848/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1787695861.623922:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695861.623928:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695861.623933:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4700. 00000020:00000040:3.0:1787695861.623939:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:3.0:1787695861.623948:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695861.623951:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695861.623958:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695861.623961:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:1.0:1787695861.623963:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1787695861.623965:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787695861.623969:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695861.623973:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702c08. 02000000:00000010:3.0:1787695861.623973:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675400. 00000100:00000001:3.0:1787695861.623988:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695861.623989:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:1.0:1787695861.623994:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000100:00100000:3.0:1787695861.623998:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4700 x1874531139535232/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695861.624002:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695861.624017:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6a00 x1874531139534848/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695861.624033:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695861.624037:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695861.624049:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008818c2cc] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4700 x1874531139535232/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695861.624063:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695861.624068:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1787695861.624069:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6a00 x1874531139534848/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695861.624072:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1787695861.624075:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695861.624079:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695861.624087:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695861.624097:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3749 00000100:00000001:1.0:1787695861.624114:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000400:00000200:3.0:1787695861.624115:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695861.624118:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787695861.624120:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695861.624122:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695861.624126:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695861.624130:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695861.624130:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695861.624139:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695861.624142:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695861.624149:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00100000:1.0:1787695861.624153:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6a00 x1874531139534848/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695861.624154:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695861.624158:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:3.0:1787695861.624165:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695861.624172:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000100:00000001:1.0:1787695861.624184:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787695861.624184:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:3.0:1787695861.624197:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:1.0:1787695861.624203:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6a00 x1874531139534848/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695861.624205:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695861.624214:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695861.624223:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695861.624234:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695861.624235:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695861.624239:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.624249:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695861.624252:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1787695861.624252:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00000001:1.0:1787695861.624260:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1787695861.624262:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695861.624264:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787695861.624265:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695861.624270:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00080000:1.0:1787695861.624271:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:3.0:1787695861.624278:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1787695861.624280:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 02000000:00000001:3.0:1787695861.624286:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.624288:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787695861.624292:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695861.624298:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000040:1.0:1787695861.624300:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6a00 x1874531139534848/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1787695861.624305:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695861.624316:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695861.624324:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5500. 00000020:00000040:3.0:1787695861.624331:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 00000100:00100000:1.0:1787695861.624332:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139534848:192.168.201.140@tcp:250:kworker.0 02000000:00000001:3.0:1787695861.624339:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695861.624342:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.624343:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695861.624345:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787695861.624347:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1787695861.624352:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1787695861.624355:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6a00 x1874531139534848/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695861.624356:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695861.624399:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 00000100:00000001:1.0:1787695861.624400:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695861.624407:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1787695861.624408:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695861.624415:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 00000100:00100000:3.0:1787695861.624422:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5500 x1874531139535360/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787695861.624424:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695861.624428:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695861.624433:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000100:00000001:3.0:1787695861.624436:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695861.624442:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787695861.624446:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000040:3.0:1787695861.624451:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a777d68] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5500 x1874531139535360/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1787695861.624453:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675800. 02000000:00000001:1.0:1787695861.624466:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1787695861.624468:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695861.624469:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695861.624476:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6a00. 00000100:00080000:3.0:1787695861.624477:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695861.624489:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00080000:3.0:1787695861.624491:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787695861.624496:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695861.624497:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787695861.624505:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695861.624507:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695861.624513:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695861.624514:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695861.624517:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.624520:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695861.624525:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1787695861.624529:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3749 00000100:00000001:1.0:1787695861.624530:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000400:00000200:3.0:1787695861.624540:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:1.0:1787695861.624542:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4a80 x1874531139534976/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695861.624548:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695861.624558:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695861.624567:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1787695861.624567:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695861.624573:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695861.624574:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787695861.624581:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695861.624587:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1787695861.624587:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695861.624591:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695861.624599:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00100000:1.0:1787695861.624600:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139534976:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695861.624605:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695861.624610:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:3.0:1787695861.624611:0:38:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1787695861.624613:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695861.624615:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 02000000:00000001:1.0:1787695861.624616:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695861.624619:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 00000020:00000001:3.0:1787695861.624621:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1787695861.624623:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695861.624628:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000400:00000200:1.0:1787695861.624630:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695861.624634:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702c08. 00000020:00000001:3.0:1787695861.624636:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000010:1.0:1787695861.624639:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000020:00000001:3.0:1787695861.624644:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000200:1.0:1787695861.624646:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139534976, portal 10 00000100:00000040:3.0:1787695861.624651:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695861.624652:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695861.624660:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930146432 : -128555779405184 : ffff8b14466f4a80) 00000100:00000001:3.0:1787695861.624661:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695861.624664:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695861.624670:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695861.624673:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:3.0:1787695861.624675:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695861.624680:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695861.624681:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4a80 x1874531139534976/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695886 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695861.624683:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695861.624687:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695861.624693:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695861.624698:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695861.624703:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695861.624710:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695861.624711:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695861.624723:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5f80. 00000020:00000040:3.0:1787695861.624734:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000100:00000001:1.0:1787695861.624735:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787695861.624740:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000040:1.0:1787695861.624742:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695861.624749:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46e58. 02000000:00000001:3.0:1787695861.624751:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695861.624753:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139534976, offset 0 00000100:00000001:3.0:1787695861.624755:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695861.624757:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1787695861.624760:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695861.624760:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695861.624771:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676000. 00000400:00000200:1.0:1787695861.624775:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695861.624784:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:3.0:1787695861.624785:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695861.624788:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695861.624793:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695861.624798:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695861.624801:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00100000:3.0:1787695861.624801:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5f80 x1874531139535488/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695861.624821:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16989:16989:256:4294967295] 192.168.201.140@tcp LPNI seq info [16988:16988:8:4294967295] 00000400:00000200:1.0:1787695861.624840:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695861.624853:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:3.0:1787695861.624854:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695861.624864:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695861.624865:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695861.624875:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 00000100:00000040:3.0:1787695861.624881:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006e7bde76] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5f80 x1874531139535488/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1787695861.624886:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695861.624891:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 8 secs later) 00000800:00000200:1.1:1787695861.624903:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695861.624911:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 8 secs later) 00000100:00000001:3.0:1787695861.624919:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787695861.624921:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695861.624928:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000100:00080000:3.0:1787695861.624928:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695861.624936:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:3.0:1787695861.624937:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695861.624943:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695861.624948:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1787695861.624950:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1787695861.624952:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1787695861.624959:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46e58 00000100:00080000:3.0:1787695861.624959:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3749 00000400:00000010:1.0:1787695861.624967:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46e58. 00000100:00000001:1.0:1787695861.624972:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:3.0:1787695861.624972:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695861.624987:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000200:1.0:1787695861.624990:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4a80 x1874531139534976/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695886 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695861.624990:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695861.625001:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695861.625006:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695861.625009:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695861.625019:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695861.625026:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695861.625029:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000020:00000001:3.0:1787695861.625035:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695861.625039:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000040:3.0:1787695861.625039:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:3.0:1787695861.625047:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:1.0:1787695861.625053:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4a80 x1874531139534976/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695886 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695861.625056:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:3.0:1787695861.625067:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695861.625072:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695861.625079:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695861.625088:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695861.625090:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00080000:3.0:1787695861.625093:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695861.625094:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695861.625102:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695861.625109:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695861.625114:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695861.625115:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.625118:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787695861.625119:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695861.625125:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787695861.625127:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787695861.625128:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695861.625132:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695861.625133:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000100:00000010:3.0:1787695861.625136:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6d80. 00000400:00000200:1.0:1787695861.625141:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000040:3.0:1787695861.625142:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=10 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000001:1.0:1787695861.625154:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298423239 02000000:00000001:3.0:1787695861.625155:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695861.625162:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.625163:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298433337 00000100:00000001:3.0:1787695861.625175:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787695861.625176:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298423239 00000100:00000001:1.0:1787695861.625184:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298433337 02000000:00000001:3.0:1787695861.625186:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787695861.625192:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695861.625193:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.625195:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.625200:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000010:3.0:1787695861.625200:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 00000100:00000001:3.0:1787695861.625212:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695861.625215:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695861/real 1787695861] req@ffff8b14466f4a80 x1874531139534976/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695861.625221:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6d80 x1874531139535616/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1787695861.625229:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695861.625234:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695861.625240:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702c08. 00000100:00000001:3.0:1787695861.625245:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1787695861.625246:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:1.0:1787695861.625252:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000100:00000001:3.0:1787695861.625252:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.625270:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:3.0:1787695861.625274:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c3be57b] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6d80 x1874531139535616/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695861.625280:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4a80 x1874531139534976/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695861.625319:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695861.625323:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4a80 x1874531139534976/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695861.625334:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695861.625340:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695861.625346:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695861.625346:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1787695861.625358:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:3.0:1787695861.625359:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695861.625386:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695861.625393:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4a80 x1874531139534976/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695861.625420:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1787695861.625422:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695861.625430:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3759) 00000100:00000040:1.0:1787695861.625431:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4a80 x1874531139534976/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.625442:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695861.625444:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.625446:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695861.625447:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.625449:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695861.625451:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695861.625453:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695861.625455:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695861.625457:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695861.625459:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4a80 x1874531139534976/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695861.625464:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139534976:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695861.625466:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695861.625469:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4000 x1874531139535104/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.625473:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695861.625474:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695861.625476:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695861.625478:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695861.625479:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695861.625481:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139535104:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695861.625484:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695861.625486:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695861.625492:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695861.625494:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695861.625497:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695861.625498:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695861.625501:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695861.625504:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702c08. 00000400:00000010:1.0:1787695861.625506:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000100:00000200:1.0:1787695861.625509:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139535104, portal 10 00000100:00000001:1.0:1787695861.625510:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695861.625511:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930143744 : -128555779407872 : ffff8b14466f4000) 00000100:00000040:1.0:1787695861.625516:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4000 x1874531139535104/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695886 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.625521:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695861.625523:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695861.625524:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46e58. 00000100:00000200:1.0:1787695861.625527:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139535104, offset 0 00000400:00000200:1.0:1787695861.625529:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695861.625532:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695861.625535:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695861.625537:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695861.625539:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695861.625541:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695861.625543:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695861.625545:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16990:16990:256:4294967295] 192.168.201.140@tcp LPNI seq info [16989:16989:8:4294967295] 00000400:00000200:1.0:1787695861.625548:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695861.625553:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695861.625556:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695861.625558:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 00000800:00000200:1.0:1787695861.625561:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695861.625564:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 8 secs later) 00000800:00000200:1.1:1787695861.625567:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695861.625569:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 8 secs later) 00000800:00000100:1.0:1787695861.625571:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695861.625576:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000400:00000200:1.0:1787695861.625579:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695861.625581:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695861.625584:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46e58 00000400:00000010:1.0:1787695861.625585:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46e58. 00000100:00000001:1.0:1787695861.625587:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695861.625589:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4000 x1874531139535104/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695886 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.625595:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695861.625598:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4000 x1874531139535104/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695886 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.625602:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.625604:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695861.625606:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695861.625609:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787695861.625611:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695861.625612:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298422088 00000100:00000001:1.0:1787695861.625614:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298433337 00000100:00000001:1.0:1787695861.625616:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298422088 00000100:00000001:1.0:1787695861.625618:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298433337 00000100:00000001:1.0:1787695861.625619:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695861.625620:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.625622:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695861.625624:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695861/real 1787695861] req@ffff8b14466f4000 x1874531139535104/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695861.625629:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695861.625630:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695861.625631:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702c08. 00000400:00000200:1.0:1787695861.625632:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:1.0:1787695861.625634:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000100:00000001:1.0:1787695861.625639:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695861.625641:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4000 x1874531139535104/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695861.625648:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4000 x1874531139535104/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.625656:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695861.625657:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.625660:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695861.625661:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695861.625665:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4000 x1874531139535104/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.625673:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695861.625677:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4000 x1874531139535104/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.625685:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695861.625688:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.625690:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695861.625691:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.625694:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695861.625696:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695861.625698:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695861.625702:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695861.625705:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695861.625708:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4000 x1874531139535104/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695861.625715:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139535104:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695861.625718:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695861.625725:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4700 x1874531139535232/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.625730:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695861.625731:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695861.625733:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695861.625734:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695861.625736:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695861.625738:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139535232:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695861.625740:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695861.625741:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695861.625743:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695861.625744:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695861.625747:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695861.625748:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695861.625751:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695861.625753:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702c08. 00000400:00000010:1.0:1787695861.625756:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000100:00000200:1.0:1787695861.625758:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139535232, portal 4 00000100:00000001:1.0:1787695861.625760:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695861.625761:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930145536 : -128555779406080 : ffff8b14466f4700) 00000100:00000040:1.0:1787695861.625765:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4700 x1874531139535232/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695886 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.625770:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695861.625772:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695861.625773:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46e58. 00000100:00000200:1.0:1787695861.625775:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139535232, offset 0 00000400:00000200:1.0:1787695861.625778:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695861.625780:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695861.625782:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695861.625784:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695861.625786:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695861.625788:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695861.625790:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695861.625795:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16991:16991:256:4294967295] 192.168.201.140@tcp LPNI seq info [16990:16990:8:4294967295] 00000400:00000200:1.0:1787695861.625798:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695861.625803:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695861.625807:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695861.625826:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 00000800:00000200:1.0:1787695861.625829:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695861.625831:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 8 secs later) 00000800:00000200:1.1:1787695861.625834:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695861.625836:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 8 secs later) 00000800:00000100:1.0:1787695861.625839:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695861.625841:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000400:00000200:1.0:1787695861.625843:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695861.625846:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695861.625848:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46e58 00000400:00000010:1.0:1787695861.625849:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46e58. 00000100:00000001:1.0:1787695861.625851:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695861.625854:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4700 x1874531139535232/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695886 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.625860:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695861.625862:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4700 x1874531139535232/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695886 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.625867:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.625868:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695861.625870:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695861.625873:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695861.625875:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695861.625877:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298422118 00000100:00000001:1.0:1787695861.625879:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298433337 00000100:00000001:1.0:1787695861.625880:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298422118 00000100:00000001:1.0:1787695861.625882:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298433337 00000100:00000001:1.0:1787695861.625883:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695861.625888:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.625890:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695861.625893:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695861/real 1787695861] req@ffff8b14466f4700 x1874531139535232/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695861.625899:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695861.625900:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695861.625901:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702c08. 00000400:00000200:1.0:1787695861.625903:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:1.0:1787695861.625904:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000100:00000001:1.0:1787695861.625906:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695861.625909:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4700 x1874531139535232/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695861.625918:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4700 x1874531139535232/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.625925:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695861.625926:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.625928:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695861.625930:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695861.625933:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4700 x1874531139535232/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.625941:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695861.625945:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4700 x1874531139535232/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.625954:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695861.625957:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.625958:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695861.625960:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.625962:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695861.625963:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695861.625965:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695861.625973:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695861.625974:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695861.625978:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4700 x1874531139535232/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695861.625983:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139535232:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695861.625986:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695861.625990:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5500 x1874531139535360/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.625997:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695861.625999:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695861.626001:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695861.626003:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695861.626005:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695861.626008:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139535360:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695861.626012:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695861.626015:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695861.626016:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695861.626018:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695861.626022:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695861.626025:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695861.626028:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695861.626031:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702c08. 00000400:00000010:1.0:1787695861.626035:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000100:00000200:1.0:1787695861.626038:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139535360, portal 4 00000100:00000001:1.0:1787695861.626041:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695861.626042:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930149120 : -128555779402496 : ffff8b14466f5500) 00000100:00000040:1.0:1787695861.626050:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5500 x1874531139535360/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695886 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626058:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695861.626070:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695861.626073:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46e58. 00000100:00000200:1.0:1787695861.626077:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139535360, offset 0 00000400:00000200:1.0:1787695861.626082:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695861.626086:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695861.626090:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695861.626095:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695861.626099:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695861.626103:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695861.626105:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695861.626109:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16992:16992:256:4294967295] 192.168.201.140@tcp LPNI seq info [16991:16991:8:4294967295] 00000400:00000200:1.0:1787695861.626114:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695861.626121:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695861.626126:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695861.626130:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 00000800:00000200:1.0:1787695861.626133:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695861.626136:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 8 secs later) 00000800:00000200:1.1:1787695861.626139:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695861.626146:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 8 secs later) 00000800:00000100:1.0:1787695861.626149:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695861.626150:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000400:00000200:1.0:1787695861.626153:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695861.626155:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695861.626158:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46e58 00000400:00000010:1.0:1787695861.626159:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46e58. 00000100:00000001:1.0:1787695861.626161:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695861.626164:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5500 x1874531139535360/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695886 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626170:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695861.626172:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5500 x1874531139535360/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695886 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626177:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.626178:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695861.626184:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695861.626186:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695861.626189:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695861.626191:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298421790 00000100:00000001:1.0:1787695861.626192:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298433337 00000100:00000001:1.0:1787695861.626194:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298421790 00000100:00000001:1.0:1787695861.626196:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298433337 00000100:00000001:1.0:1787695861.626198:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695861.626199:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.626200:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695861.626203:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695861/real 1787695861] req@ffff8b14466f5500 x1874531139535360/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695861.626208:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695861.626208:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695861.626210:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702c08. 00000400:00000200:1.0:1787695861.626211:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:1.0:1787695861.626213:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000100:00000001:1.0:1787695861.626214:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695861.626216:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5500 x1874531139535360/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695861.626222:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5500 x1874531139535360/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626226:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695861.626227:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.626229:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695861.626229:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695861.626232:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5500 x1874531139535360/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626235:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695861.626238:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5500 x1874531139535360/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626249:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695861.626251:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.626253:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695861.626255:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.626257:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695861.626258:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695861.626261:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695861.626265:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695861.626268:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695861.626272:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5500 x1874531139535360/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695861.626280:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139535360:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695861.626285:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.626287:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695861.626291:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4a80 x1874531139534976/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626297:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695861.626299:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695861.626302:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695861.626305:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695861.626307:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695861.626309:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695861.626311:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695861.626313:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:1.0:1787695861.626319:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695861.626320:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695861.626322:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4a80. 00000100:00000001:1.0:1787695861.626324:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695861.626326:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.626333:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695861.626338:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4000 x1874531139535104/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626345:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695861.626347:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695861.626349:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695861.626354:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695861.626355:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695861.626356:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695861.626358:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695861.626359:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:1.0:1787695861.626380:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695861.626382:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695861.626383:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4000. 00000100:00000001:1.0:1787695861.626385:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695861.626386:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.626388:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695861.626392:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4700 x1874531139535232/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626398:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695861.626400:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695861.626403:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695861.626406:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695861.626407:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695861.626408:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695861.626409:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695861.626411:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675400. 02000000:00000001:1.0:1787695861.626415:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695861.626416:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695861.626418:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4700. 00000100:00000001:1.0:1787695861.626421:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695861.626423:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.626424:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695861.626427:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5500 x1874531139535360/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626440:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695861.626441:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695861.626442:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695861.626444:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695861.626445:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695861.626446:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695861.626447:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695861.626448:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:1.0:1787695861.626450:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695861.626451:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695861.626452:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5500. 00000100:00000001:1.0:1787695861.626454:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695861.626454:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.626455:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.626458:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695861.626459:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.626461:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695861.626463:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695861.626464:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695861.626467:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5f80 x1874531139535488/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626472:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695861.626474:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695861.626475:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695861.626477:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695861.626479:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695861.626481:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139535488:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695861.626483:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695861.626484:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695861.626486:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695861.626487:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695861.626489:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695861.626490:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695861.626496:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695861.626498:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702c08. 00000400:00000010:1.0:1787695861.626500:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000100:00000200:1.0:1787695861.626503:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139535488, portal 10 00000100:00000001:1.0:1787695861.626504:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695861.626505:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930151808 : -128555779399808 : ffff8b14466f5f80) 00000100:00000040:1.0:1787695861.626510:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5f80 x1874531139535488/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695886 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626515:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695861.626517:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695861.626518:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46e58. 00000100:00000200:1.0:1787695861.626520:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139535488, offset 0 00000400:00000200:1.0:1787695861.626523:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695861.626525:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695861.626527:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695861.626529:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695861.626531:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695861.626533:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695861.626535:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695861.626537:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16993:16993:256:4294967295] 192.168.201.140@tcp LPNI seq info [16992:16992:8:4294967295] 00000400:00000200:1.0:1787695861.626540:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695861.626545:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695861.626547:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695861.626549:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 00000800:00000200:1.0:1787695861.626552:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695861.626554:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 8 secs later) 00000800:00000200:1.1:1787695861.626557:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695861.626559:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 8 secs later) 00000800:00000100:1.0:1787695861.626562:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695861.626564:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000400:00000200:1.0:1787695861.626566:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695861.626568:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695861.626570:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46e58 00000400:00000010:1.0:1787695861.626576:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46e58. 00000100:00000001:1.0:1787695861.626578:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695861.626581:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5f80 x1874531139535488/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695886 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626587:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695861.626589:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5f80 x1874531139535488/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695886 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626594:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.626595:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695861.626597:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695861.626600:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695861.626602:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695861.626604:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298421977 00000100:00000001:1.0:1787695861.626606:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298433337 00000100:00000001:1.0:1787695861.626607:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298421977 00000100:00000001:1.0:1787695861.626609:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298433337 00000100:00000001:1.0:1787695861.626610:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695861.626612:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.626613:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695861.626615:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695861/real 1787695861] req@ffff8b14466f5f80 x1874531139535488/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695861.626620:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695861.626621:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695861.626623:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702c08. 00000400:00000200:1.0:1787695861.626624:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:1.0:1787695861.626625:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000100:00000001:1.0:1787695861.626627:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695861.626629:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5f80 x1874531139535488/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695861.626634:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5f80 x1874531139535488/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626642:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695861.626643:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.626644:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695861.626645:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695861.626649:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5f80 x1874531139535488/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626656:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695861.626661:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5f80 x1874531139535488/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626668:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695861.626671:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.626672:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695861.626674:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.626676:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695861.626677:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695861.626680:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695861.626684:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695861.626687:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695861.626691:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5f80 x1874531139535488/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695861.626701:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139535488:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695861.626705:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695861.626709:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6d80 x1874531139535616/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626714:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695861.626715:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695861.626722:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695861.626723:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695861.626725:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695861.626727:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139535616:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695861.626729:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695861.626731:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695861.626732:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695861.626734:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695861.626735:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695861.626737:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695861.626740:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695861.626741:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702c08. 00000400:00000010:1.0:1787695861.626744:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000100:00000200:1.0:1787695861.626746:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139535616, portal 10 00000100:00000001:1.0:1787695861.626748:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695861.626749:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930155392 : -128555779396224 : ffff8b14466f6d80) 00000100:00000040:1.0:1787695861.626754:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6d80 x1874531139535616/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695886 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626758:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695861.626760:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695861.626761:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46e58. 00000100:00000200:1.0:1787695861.626763:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139535616, offset 0 00000400:00000200:1.0:1787695861.626766:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695861.626768:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695861.626770:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695861.626772:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695861.626774:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695861.626776:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695861.626778:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695861.626780:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16994:16994:256:4294967295] 192.168.201.140@tcp LPNI seq info [16993:16993:8:4294967295] 00000400:00000200:1.0:1787695861.626783:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695861.626787:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695861.626794:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695861.626796:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 00000800:00000200:1.0:1787695861.626799:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695861.626801:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 8 secs later) 00000800:00000200:1.1:1787695861.626804:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695861.626806:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 8 secs later) 00000800:00000100:1.0:1787695861.626822:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695861.626823:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000400:00000200:1.0:1787695861.626826:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695861.626828:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695861.626831:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46e58 00000400:00000010:1.0:1787695861.626832:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46e58. 00000100:00000001:1.0:1787695861.626834:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695861.626837:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6d80 x1874531139535616/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695886 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626842:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695861.626844:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6d80 x1874531139535616/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695886 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626849:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.626850:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695861.626852:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695861.626855:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695861.626857:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695861.626858:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298421955 00000100:00000001:1.0:1787695861.626860:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298433337 00000100:00000001:1.0:1787695861.626862:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298421955 00000100:00000001:1.0:1787695861.626864:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298433337 00000100:00000001:1.0:1787695861.626865:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695861.626867:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.626868:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695861.626870:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695861/real 1787695861] req@ffff8b14466f6d80 x1874531139535616/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695861.626880:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695861.626881:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695861.626883:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702c08. 00000400:00000200:1.0:1787695861.626884:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:1.0:1787695861.626885:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000100:00000001:1.0:1787695861.626887:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695861.626889:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6d80 x1874531139535616/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695861.626895:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6d80 x1874531139535616/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626900:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695861.626901:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.626902:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695861.626903:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695861.626905:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6d80 x1874531139535616/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626910:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695861.626914:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6d80 x1874531139535616/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626921:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695861.626923:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.626924:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695861.626925:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.626927:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695861.626929:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695861.626932:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695861.626935:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695861.626938:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695861.626948:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6d80 x1874531139535616/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695861.626958:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139535616:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695861.626962:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.626964:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695861.626967:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5f80 x1874531139535488/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.626972:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695861.626973:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695861.626975:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695861.626977:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695861.626978:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695861.626979:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695861.626980:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695861.626981:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676000. 02000000:00000001:1.0:1787695861.626983:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695861.626985:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695861.626986:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5f80. 00000100:00000001:1.0:1787695861.626989:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695861.626990:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.626992:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695861.626995:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6d80 x1874531139535616/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695886 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695861.627001:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695861.627003:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695861.627005:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695861.627008:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695861.627009:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695861.627011:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695861.627013:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695861.627014:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:1.0:1787695861.627018:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695861.627024:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695861.627026:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6d80. 00000100:00000001:1.0:1787695861.627029:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695861.627031:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.627033:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695861.627037:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695861.627038:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695861.627040:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695861.627041:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695861.686442:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695861.686451:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695861.686453:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695861.686459:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695861.686463:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695861.686467:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695861.686470:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695861.686472:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695861.686476:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695862.134444:0:2403:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695862.134456:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695862.134459:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695862.134464:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695862.134467:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695862.134470:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695862.134474:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695862.326427:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695862.326434:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695862.326440:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695862.326442:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695862.326446:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695862.326447:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695862.326451:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695862.326452:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695862.326456:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695862.326457:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695862.326460:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695862.326462:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695862.326549:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695862.326552:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695862.326555:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787695862.647925:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695862.647942:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695862.647944:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695862.647949:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695862.647951:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695862.647954:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695862.647958:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695862.710546:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695862.710555:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695862.710557:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695862.710562:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695862.710566:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695862.710569:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695862.710571:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695862.710574:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695862.710576:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695863.158452:0:2403:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695863.158463:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695863.158464:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695863.158469:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695863.158471:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695863.158491:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695863.158494:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695863.350476:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695863.350485:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695863.350487:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695863.350490:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695863.350496:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695863.350498:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695863.350500:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695863.350502:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695863.350506:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695863.350508:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695863.350509:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695863.350511:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695863.350513:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695863.350514:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695863.350516:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695863.350518:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695863.350522:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695863.350524:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695863.350527:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695863.350529:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695863.350532:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695863.350533:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695863.350536:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695863.350538:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695863.350591:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695863.350595:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695863.350599:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787695863.670437:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695863.670457:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695863.670460:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695863.670466:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695863.670469:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695863.670474:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695863.670478:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695863.734436:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695863.734447:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695863.734449:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695863.734453:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695863.734457:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695863.734461:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695863.734463:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695863.734466:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695863.734487:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695864.182431:0:2403:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695864.182442:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695864.182444:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695864.182449:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695864.182451:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695864.182454:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695864.182475:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695864.374467:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695864.374472:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695864.374477:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695864.374478:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695864.374481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695864.374482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695864.374484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695864.374485:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695864.374488:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695864.374489:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695864.374491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695864.374492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695864.374581:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695864.374585:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695864.374587:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787695864.694443:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695864.694462:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695864.694465:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695864.694472:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695864.694475:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695864.694479:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695864.694483:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695864.758502:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695864.758512:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695864.758515:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695864.758523:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695864.758527:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695864.758536:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695864.758540:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695864.758543:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695864.758546:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695865.206530:0:2403:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695865.206537:0:2403:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000400:1.0:1787695865.206552:0:2403:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695780/real 1787695780] req@ffff8b1446667b80 x1874531139520896/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695865 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695865.206570:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695865.206573:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695865.206577:0:2403:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702630. 00000400:00000200:1.0:1787695865.206585:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695865.206587:0:2403:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695865.206592:0:2403:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695865.206596:0:2403:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446667b80 x1874531139520896/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695865 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695865.206605:0:2403:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446667b80 x1874531139520896/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695865 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695865.206612:0:2403:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695865.206633:0:2403:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695865.206636:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695865.206638:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695865.206643:0:2403:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446667b80 x1874531139520896/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695865 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695865.206649:0:2403:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:1.0:1787695865.206652:0:2403:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b1458081800 already not connected (conn 31, was 15): DISCONN 00000100:00000001:1.0:1787695865.206655:0:2403:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:1.0:1787695865.206657:0:2403:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695865.206664:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695865.206665:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695865.206668:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695865.206670:0:2403:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695865.206672:0:2403:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695865.206674:0:2403:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695865.206676:0:2403:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695865.206680:0:2403:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446667b80 x1874531139520896/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695865 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695865.206686:0:2403:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:1.0:1787695865.206689:0:2403:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695865.206690:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695865.206691:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695865.206695:0:2403:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446667b80 x1874531139520896/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695865 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695865.206701:0:2403:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446667b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2403:1874531139520896:192.168.201.140@tcp:400:kworker.0 00000100:00000001:1.0:1787695865.206706:0:2403:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695865.206708:0:2403:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695865.206711:0:2403:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446667b80 x1874531139520896/t0(0) o400->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695865 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695865.206717:0:2403:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695865.206720:0:2403:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695865.206728:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1447532c00. 02000000:00000001:1.0:1787695865.206732:0:2403:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695865.206735:0:2403:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695865.206737:0:2403:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695865.206739:0:2403:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695865.206742:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1445678000. 02000000:00000001:1.0:1787695865.206761:0:2403:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695865.206763:0:2403:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695865.206765:0:2403:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446667b80. 00000100:00000001:1.0:1787695865.206770:0:2403:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695865.206771:0:2403:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695865.206773:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695865.206776:0:2403:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695865.206777:0:2403:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695865.206779:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695865.206782:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695865.398489:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695865.398495:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695865.398501:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695865.398502:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695865.398506:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695865.398508:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695865.398512:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695865.398513:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695865.398517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695865.398518:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695865.398521:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695865.398523:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695865.398589:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695865.398592:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695865.398595:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787695865.718477:0:2404:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1787695865.718484:0:2404:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000400:1.0:1787695865.718505:0:2404:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1787695780/real 1787695780] req@ffff8b1446665f80 x1874531139521024/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695865 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695865.718523:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695865.718526:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695865.718530:0:2404:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144617b2c0. 00000400:00000200:1.0:1787695865.718536:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1447f964c8 00000400:00000010:1.0:1787695865.718539:0:2404:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1447f964c8. 00000100:00000001:1.0:1787695865.718551:0:2404:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695865.718556:0:2404:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665f80 x1874531139521024/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695865 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695865.718566:0:2404:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665f80 x1874531139521024/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695865 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695865.718572:0:2404:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695865.718575:0:2404:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695865.718578:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695865.718580:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695865.718584:0:2404:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8b1446665f80 x1874531139521024/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695865 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695865.718592:0:2404:0:(import.c:429:ptlrpc_fail_import()) Process entered 00000100:00080000:1.0:1787695865.718595:0:2404:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8b144515c000 already not connected (conn 27, was 11): DISCONN 00000100:00000001:1.0:1787695865.718599:0:2404:0:(import.c:436:ptlrpc_fail_import()) Process leaving 00000100:00000001:1.0:1787695865.718600:0:2404:0:(client.c:2555:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695865.718608:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695865.718610:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695865.718613:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695865.718615:0:2404:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695865.718618:0:2404:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695865.718620:0:2404:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695865.718623:0:2404:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695865.718628:0:2404:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665f80 x1874531139521024/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695865 ref 1 fl Rpc:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695865.718636:0:2404:0:(client.c:2190:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000100:00000001:1.0:1787695865.718639:0:2404:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695865.718641:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695865.718643:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695865.718647:0:2404:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665f80 x1874531139521024/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695865 ref 1 fl Interpret:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695865.718656:0:2404:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:608c526c-eb6e-4a54-80a5-8969828241eb:2404:1874531139521024:192.168.201.140@tcp:400:kworker.0 00000100:00000001:1.0:1787695865.718668:0:2404:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695865.718671:0:2404:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695865.718676:0:2404:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665f80 x1874531139521024/t0(0) o400->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 224/224 e 0 to 1 dl 1787695865 ref 1 fl Complete:XNQU/200/ffffffff rc -11/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695865.718683:0:2404:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695865.718686:0:2404:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695865.718690:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff8b1447625600. 02000000:00000001:1.0:1787695865.718694:0:2404:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695865.718697:0:2404:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695865.718700:0:2404:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695865.718702:0:2404:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695865.718705:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff8b1445678800. 02000000:00000001:1.0:1787695865.718709:0:2404:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695865.718710:0:2404:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695865.718712:0:2404:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665f80. 00000100:00000001:1.0:1787695865.718716:0:2404:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695865.718718:0:2404:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695865.718720:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695865.718724:0:2404:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695865.718726:0:2404:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695865.718728:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695865.718730:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695865.782437:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695865.782444:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695865.782445:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695865.782450:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695865.782453:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695865.782456:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695865.782458:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695865.782460:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695865.782462:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695866.422469:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695866.422475:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695866.422479:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695866.422480:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695866.422483:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695866.422486:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695866.422488:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695866.422492:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695866.422494:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695866.422495:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695866.422498:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695866.422500:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695866.422501:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695866.422503:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695866.422523:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695866.422524:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695866.422525:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695866.422527:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695866.422528:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695866.422529:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695866.422530:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695866.422531:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695866.422532:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695866.422533:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695866.422535:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695866.422536:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695866.422538:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695866.422539:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695866.422540:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695866.422541:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695866.422542:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695866.422542:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695866.422543:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695866.422544:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695866.422545:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695866.422546:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695866.422627:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695866.422631:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695866.422633:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787695866.742537:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695866.742548:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787695866.742551:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695866.742554:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695866.742558:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695866.742576:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3754 00000400:00000200:3.0:1787695866.742586:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695866.742593:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695866.742596:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695866.742600:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.742603:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695866.742606:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.742608:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695866.742646:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695866.742647:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:3.0:1787695866.742650:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695866.742652:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787695866.742654:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787695866.742656:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787695866.742658:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.742659:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695866.742661:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.742662:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695866.742664:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787695866.742667:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695866.742669:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695866.742671:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695866.742674:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695866.742676:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695866.742677:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787695866.742679:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695866.742684:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4380. 00000020:00000040:3.0:1787695866.742687:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787695866.742690:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695866.742691:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695866.742693:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695866.742696:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695866.742701:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695866.742709:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674800. 00000100:00000001:3.0:1787695866.742715:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695866.742747:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4380 x1874531139535744/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695866.742759:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695866.742763:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695866.742768:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d2982c15] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4380 x1874531139535744/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695866.742801:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695866.742805:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695866.742809:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695866.742811:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695866.742813:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695866.742816:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695866.742819:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3754 00000400:00000200:3.0:1787695866.742824:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695866.742828:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695866.742830:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695866.742834:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.742836:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695866.742838:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.742840:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695866.742842:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695866.742844:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:3.0:1787695866.742845:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695866.742847:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000100:00000001:1.0:1787695866.742848:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787695866.742849:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:3.0:1787695866.742855:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000001:1.0:1787695866.742858:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1787695866.742859:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695866.742864:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787695866.742864:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695866.742877:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695866.742885:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4380 x1874531139535744/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695866.742887:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695866.742899:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00000001:1.0:1787695866.742909:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695866.742912:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695866.742914:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000001:1.0:1787695866.742917:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787695866.742920:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695866.742922:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695866.742926:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695866.742930:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695866.742937:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1787695866.742938:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139535744:192.168.201.140@tcp:250:kworker.0 02000000:00000001:3.0:1787695866.742940:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695866.742946:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695866.742953:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:3.0:1787695866.742953:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695866.742961:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695866.742963:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695866.742966:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:3.0:1787695866.742966:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5180. 00000020:00000040:3.0:1787695866.742973:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:1.0:1787695866.742975:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695866.742977:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695866.742980:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695866.742983:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787695866.742983:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695866.742986:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695866.742988:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695866.742992:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 00000100:00000001:3.0:1787695866.742995:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695866.743004:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5180 x1874531139535872/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:1.0:1787695866.743005:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695866.743017:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695866.743024:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695866.743027:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695866.743030:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a115abf4] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5180 x1874531139535872/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695866.743039:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695866.743054:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695866.743056:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702630. 00000100:00080000:3.0:1787695866.743058:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695866.743066:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695866.743068:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695866.743070:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695866.743075:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695866.743080:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3754 00000400:00000010:1.0:1787695866.743082:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000400:00000200:3.0:1787695866.743088:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695866.743094:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000200:1.0:1787695866.743096:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139535744, portal 25 00000100:00080000:3.0:1787695866.743098:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695866.743101:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695866.743102:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930144640 : -128555779406976 : ffff8b14466f4380) 00000100:00000040:1.0:1787695866.743112:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4380 x1874531139535744/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695891 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695866.743113:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.743127:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695866.743130:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695866.743138:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695866.743138:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000010:1.0:1787695866.743142:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000100:00000001:3.0:1787695866.743146:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000200:1.0:1787695866.743148:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139535744, offset 0 00000020:00000001:3.0:1787695866.743157:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695866.743160:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000400:00000200:1.0:1787695866.743162:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000020:00000001:3.0:1787695866.743167:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695866.743171:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000400:00000200:1.0:1787695866.743182:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:3.0:1787695866.743184:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:3.0:1787695866.743188:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:3.0:1787695866.743195:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695866.743196:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:3.0:1787695866.743206:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695866.743210:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1787695866.743211:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.743217:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695866.743220:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000020:00000040:3.0:1787695866.743225:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000400:00000200:1.0:1787695866.743229:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00080000:3.0:1787695866.743237:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:1.0:1787695866.743238:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.743248:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695866.743253:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1787695866.743259:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16995:16995:256:4294967295] 192.168.201.140@tcp LPNI seq info [16994:16994:8:4294967295] 02000000:00000001:3.0:1787695866.743259:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695866.743268:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695866.743275:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695866.743279:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1787695866.743285:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000008:00000001:3.0:1787695866.743285:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695866.743294:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695866.743300:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6d80. 00000020:00000040:3.0:1787695866.743303:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695866.743307:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695866.743308:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695866.743310:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695866.743311:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695866.743315:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695866.743319:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 02000000:00000010:3.0:1787695866.743321:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676000. 00000800:00000200:1.0:1787695866.743332:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695866.743333:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1787695866.743347:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 00000100:00100000:3.0:1787695866.743349:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6d80 x1874531139536000/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695866.743359:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:1.0:1787695866.743392:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695866.743398:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 3 secs later) 00000800:00000200:1.1:1787695866.743404:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695866.743406:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 3 secs later) 00000800:00000100:1.0:1787695866.743410:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695866.743412:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000400:00000200:1.0:1787695866.743420:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695866.743431:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695866.743437:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:1.0:1787695866.743441:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000100:00000001:3.0:1787695866.743442:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.743452:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1787695866.743453:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c3be57b] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6d80 x1874531139536000/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695866.743464:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4380 x1874531139535744/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695891 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695866.743465:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695866.743473:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695866.743482:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787695866.743484:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695866.743487:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695866.743492:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695866.743499:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3754 00000100:00000001:1.0:1787695866.743512:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1787695866.743513:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:1.0:1787695866.743522:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4380 x1874531139535744/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695891 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695866.743523:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695866.743530:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695866.743539:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.743550:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695866.743556:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.743558:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695866.743561:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695866.743564:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000100:00000001:1.0:1787695866.743569:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695866.743569:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695866.743577:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:3.0:1787695866.743579:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:3.0:1787695866.743584:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:1.0:1787695866.743585:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695866.743592:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000400:00000200:1.0:1787695866.743594:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695866.743600:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298438457 00000100:00000040:3.0:1787695866.743606:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.743611:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695866.743612:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298433337 00000100:00000001:1.0:1787695866.743618:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298438457 00000100:00000040:3.0:1787695866.743622:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.743626:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695866.743628:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298433337 00000020:00000040:3.0:1787695866.743629:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:3.0:1787695866.743638:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695866.743641:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695866.743641:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695866.743652:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695866.743655:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695866.743656:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695866.743665:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.743666:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787695866.743668:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695866.743675:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787695866.743678:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695866.743683:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695866/real 1787695866] req@ffff8b14466f4380 x1874531139535744/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695866.743683:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695866.743696:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5f80. 00000020:00000040:3.0:1787695866.743705:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000400:00000001:1.0:1787695866.743708:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1787695866.743709:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000001:1.0:1787695866.743711:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1787695866.743712:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695866.743714:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702630. 00000100:00000001:3.0:1787695866.743716:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695866.743730:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695866.743732:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695866.743736:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 02000000:00000010:3.0:1787695866.743737:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 00000400:00000010:1.0:1787695866.743745:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:3.0:1787695866.743747:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.743751:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:3.0:1787695866.743756:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5f80 x1874531139536128/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1787695866.743758:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4380 x1874531139535744/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695866.743818:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4380 x1874531139535744/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695866.743820:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695866.743825:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695866.743842:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006e7bde76] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5f80 x1874531139536128/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.743843:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695866.743847:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.743857:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695866.743862:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695866.743871:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4380 x1874531139535744/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695866.743874:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.743892:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695866.743892:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1787695866.743902:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4380 x1874531139535744/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695866.743902:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1787695866.743909:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787695866.743912:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695866.743915:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.743916:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695866.743922:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695866.743923:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695866.743930:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787695866.743931:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695866.743935:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787695866.743942:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3754 00000100:00080000:1.0:1787695866.743943:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695866.743954:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000400:00000200:3.0:1787695866.743954:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695866.743964:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000400:00000200:3.0:1787695866.743970:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695866.743975:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695866.743976:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4380 x1874531139535744/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695866.743989:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.744002:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1787695866.744008:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139535744:192.168.201.140@tcp:250:kworker.0 00000100:00000040:3.0:1787695866.744011:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.744019:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695866.744025:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695866.744026:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:3.0:1787695866.744027:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695866.744029:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1787695866.744030:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:3.0:1787695866.744037:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000001:1.0:1787695866.744038:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1787695866.744042:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:1.0:1787695866.744052:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4380 x1874531139535744/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695866.744052:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.744059:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695866.744065:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.744080:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695866.744084:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695866.744090:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000040:3.0:1787695866.744090:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 02000000:00000010:1.0:1787695866.744102:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546000. 00000100:00080000:3.0:1787695866.744103:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000001:1.0:1787695866.744109:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1787695866.744110:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787695866.744113:0:2402:0:(genops.c:1217:class_import_put()) Process entered 02000000:00000001:3.0:1787695866.744116:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000020:00000040:1.0:1787695866.744117:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:1.0:1787695866.744123:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695866.744127:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674800. 02000000:00000001:3.0:1787695866.744128:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695866.744133:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787695866.744134:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695866.744136:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787695866.744142:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000010:1.0:1787695866.744143:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4380. 00000100:00000001:1.0:1787695866.744152:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000008:00100000:3.0:1787695866.744153:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787695866.744155:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.744161:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1787695866.744163:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695866.744166:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.744171:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695866.744173:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.744183:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:3.0:1787695866.744184:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5500. 00000100:00000001:1.0:1787695866.744186:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695866.744192:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1787695866.744193:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:3.0:1787695866.744199:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000040:1.0:1787695866.744201:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5180 x1874531139535872/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695866.744201:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695866.744212:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695866.744216:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695866.744224:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.744225:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695866.744231:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695866.744234:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675400. 02000000:00000001:1.0:1787695866.744235:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695866.744238:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695866.744243:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695866.744244:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695866.744251:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139535872:192.168.201.140@tcp:38:kworker.0 00000100:00100000:3.0:1787695866.744258:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5500 x1874531139536256/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695866.744259:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695866.744264:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695866.744267:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695866.744275:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695866.744282:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546000. 00000100:00000001:3.0:1787695866.744283:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787695866.744289:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695866.744290:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695866.744303:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695866.744306:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a777d68] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5500 x1874531139536256/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1787695866.744307:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702630. 00000400:00000010:1.0:1787695866.744315:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695866.744323:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139535872, portal 10 00000100:00000001:3.0:1787695866.744329:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.744332:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787695866.744334:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695866.744335:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930148224 : -128555779403392 : ffff8b14466f5180) 00000100:00080000:3.0:1787695866.744345:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695866.744348:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695866.744350:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1787695866.744352:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5180 x1874531139535872/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695891 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695866.744354:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695866.744361:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3754 00000100:00000001:1.0:1787695866.744391:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695866.744396:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000200:3.0:1787695866.744402:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695866.744404:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000100:00000200:1.0:1787695866.744409:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139535872, offset 0 00000400:00000200:3.0:1787695866.744409:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695866.744422:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1787695866.744431:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695866.744436:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.744439:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695866.744440:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695866.744446:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.744452:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695866.744453:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000020:00000001:3.0:1787695866.744455:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:1.0:1787695866.744460:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:3.0:1787695866.744468:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:3.0:1787695866.744471:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:1.0:1787695866.744473:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000020:00000040:3.0:1787695866.744474:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000020:00000001:3.0:1787695866.744481:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000020:00000001:3.0:1787695866.744484:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000200:1.0:1787695866.744486:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000040:3.0:1787695866.744490:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.744494:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695866.744499:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000040:3.0:1787695866.744499:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.744505:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695866.744512:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16996:16996:256:4294967295] 192.168.201.140@tcp LPNI seq info [16995:16995:8:4294967295] 00000020:00000040:3.0:1787695866.744513:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:3.0:1787695866.744523:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695866.744530:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695866.744531:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787695866.744542:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695866.744547:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695866.744556:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695866.744562:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:3.0:1787695866.744563:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000200:1.0:1787695866.744573:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000008:00100000:3.0:1787695866.744573:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000800:00000010:1.0:1787695866.744581:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 00000008:00000001:3.0:1787695866.744584:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695866.744588:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787695866.744593:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695866.744594:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 3 secs later) 00000800:00000200:1.1:1787695866.744603:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000010:3.0:1787695866.744604:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4700. 00000800:00000200:1.1:1787695866.744606:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 3 secs later) 00000020:00000040:3.0:1787695866.744614:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000800:00000100:1.0:1787695866.744621:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695866.744622:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695866.744625:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1787695866.744628:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000100:00000001:3.0:1787695866.744629:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695866.744632:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695866.744637:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695866.744640:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000010:3.0:1787695866.744642:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 00000100:00000001:3.0:1787695866.744647:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695866.744653:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00100000:3.0:1787695866.744659:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4700 x1874531139536384/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695866.744663:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:1.0:1787695866.744672:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000100:00000001:3.0:1787695866.744672:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695866.744678:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.744679:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695866.744686:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5180 x1874531139535872/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695891 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695866.744695:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008818c2cc] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4700 x1874531139536384/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695866.744708:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695866.744715:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695866.744747:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787695866.744748:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787695866.744750:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695866.744753:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1787695866.744757:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5180 x1874531139535872/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695891 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695866.744759:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695866.744768:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3754 00000100:00000001:1.0:1787695866.744781:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.744785:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:3.0:1787695866.744787:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695866.744795:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695866.744797:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695866.744801:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695866.744803:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000100:00000040:3.0:1787695866.744810:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695866.744811:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695866.744815:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298428359 00000100:00000001:3.0:1787695866.744820:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695866.744821:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298438457 00000100:00000001:1.0:1787695866.744826:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298428359 00000100:00000001:1.0:1787695866.744830:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298438457 00000100:00000040:3.0:1787695866.744831:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695866.744833:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695866.744840:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.744845:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787695866.744845:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00100000:1.0:1787695866.744853:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695866/real 1787695866] req@ffff8b14466f5180 x1874531139535872/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695866.744855:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695866.744859:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000400:00000001:1.0:1787695866.744868:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:3.0:1787695866.744869:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000001:1.0:1787695866.744871:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695866.744874:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702630. 00000020:00000040:3.0:1787695866.744875:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000400:00000200:1.0:1787695866.744877:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695866.744881:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000020:00000001:3.0:1787695866.744884:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:1.0:1787695866.744885:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695866.744888:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5180 x1874531139535872/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695866.744896:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000200:1.0:1787695866.744900:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5180 x1874531139535872/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.744913:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:3.0:1787695866.744913:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695866.744919:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695866.744920:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.744923:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695866.744925:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695866.744929:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00100000:1.0:1787695866.744932:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5180 x1874531139535872/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695866.744938:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695866.744948:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00000001:1.0:1787695866.744955:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695866.744960:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000040:1.0:1787695866.744966:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5180 x1874531139535872/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695866.744967:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.744973:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695866.744979:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787695866.744979:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695866.744986:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695866.744988:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787695866.744988:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695866.744994:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:3.0:1787695866.744995:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.744999:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787695866.744999:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695866.745006:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787695866.745007:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00000001:3.0:1787695866.745017:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695866.745021:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695866.745023:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000010:3.0:1787695866.745025:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4000. 00000020:00000040:3.0:1787695866.745031:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000001:1.0:1787695866.745032:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1787695866.745034:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695866.745036:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695866.745040:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695866.745041:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695866.745046:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695866.745047:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5180 x1874531139535872/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1787695866.745050:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 00000100:00000001:3.0:1787695866.745056:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695866.745068:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4000 x1874531139536512/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787695866.745074:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139535872:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695866.745085:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695866.745092:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6d80 x1874531139536000/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695866.745111:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695866.745112:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695866.745117:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.745118:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695866.745126:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695866.745129:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1787695866.745132:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5a8f356] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4000 x1874531139536512/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695866.745134:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695866.745145:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695866.745153:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787695866.745158:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139536000:192.168.201.140@tcp:38:kworker.0 00000100:00000040:3.0:1787695866.745161:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695866.745168:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695866.745170:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1787695866.745177:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 02000000:00000001:1.0:1787695866.745179:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695866.745183:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695866.745184:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3764) 02000000:00000001:1.0:1787695866.745190:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695866.745196:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695866.745199:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695866.745204:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695866.745208:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702630. 00000400:00000010:1.0:1787695866.745212:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695866.745215:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139536000, portal 10 00000100:00000001:1.0:1787695866.745217:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695866.745219:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930155392 : -128555779396224 : ffff8b14466f6d80) 00000100:00000040:1.0:1787695866.745226:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6d80 x1874531139536000/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695891 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.745231:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695866.745233:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695866.745235:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000100:00000200:1.0:1787695866.745237:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139536000, offset 0 00000400:00000200:1.0:1787695866.745240:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695866.745245:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695866.745249:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695866.745252:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695866.745255:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695866.745258:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695866.745260:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695866.745264:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16997:16997:256:4294967295] 192.168.201.140@tcp LPNI seq info [16996:16996:8:4294967295] 00000400:00000200:1.0:1787695866.745269:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695866.745277:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695866.745282:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695866.745286:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 00000800:00000200:1.0:1787695866.745294:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695866.745297:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 3 secs later) 00000800:00000200:1.1:1787695866.745301:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695866.745303:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 3 secs later) 00000800:00000100:1.0:1787695866.745305:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695866.745307:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000400:00000200:1.0:1787695866.745310:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695866.745312:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695866.745315:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:1.0:1787695866.745317:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000100:00000001:1.0:1787695866.745319:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695866.745322:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6d80 x1874531139536000/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695891 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.745328:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695866.745330:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6d80 x1874531139536000/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695891 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.745335:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.745336:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695866.745339:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695866.745342:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787695866.745344:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695866.745346:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298427207 00000100:00000001:1.0:1787695866.745348:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298438457 00000100:00000001:1.0:1787695866.745349:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298427207 00000100:00000001:1.0:1787695866.745351:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298438457 00000100:00000001:1.0:1787695866.745352:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695866.745354:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.745355:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695866.745357:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695866/real 1787695866] req@ffff8b14466f6d80 x1874531139536000/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695866.745378:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695866.745384:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695866.745385:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702630. 00000400:00000200:1.0:1787695866.745387:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695866.745389:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695866.745394:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695866.745397:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6d80 x1874531139536000/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695866.745404:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6d80 x1874531139536000/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.745413:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695866.745414:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.745416:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695866.745417:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695866.745419:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6d80 x1874531139536000/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.745424:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695866.745428:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6d80 x1874531139536000/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.745433:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695866.745435:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.745437:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695866.745438:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.745440:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695866.745442:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695866.745445:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695866.745450:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695866.745452:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695866.745456:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6d80 x1874531139536000/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695866.745476:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139536000:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695866.745481:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695866.745485:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5f80 x1874531139536128/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.745493:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695866.745495:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695866.745497:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695866.745499:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695866.745502:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695866.745505:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139536128:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695866.745507:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695866.745509:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695866.745510:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695866.745512:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695866.745515:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695866.745516:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695866.745519:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695866.745521:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702630. 00000400:00000010:1.0:1787695866.745524:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695866.745527:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139536128, portal 4 00000100:00000001:1.0:1787695866.745528:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695866.745529:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930151808 : -128555779399808 : ffff8b14466f5f80) 00000100:00000040:1.0:1787695866.745534:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5f80 x1874531139536128/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695891 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.745539:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695866.745541:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695866.745542:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000100:00000200:1.0:1787695866.745544:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139536128, offset 0 00000400:00000200:1.0:1787695866.745547:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695866.745554:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695866.745556:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695866.745559:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695866.745561:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695866.745563:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695866.745564:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695866.745566:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16998:16998:256:4294967295] 192.168.201.140@tcp LPNI seq info [16997:16997:8:4294967295] 00000400:00000200:1.0:1787695866.745569:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695866.745574:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695866.745577:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695866.745579:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 00000800:00000200:1.0:1787695866.745581:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695866.745584:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 3 secs later) 00000800:00000200:1.1:1787695866.745588:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695866.745591:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 3 secs later) 00000800:00000100:1.0:1787695866.745595:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695866.745597:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000400:00000200:1.0:1787695866.745600:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695866.745604:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695866.745607:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:1.0:1787695866.745609:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000100:00000001:1.0:1787695866.745611:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695866.745615:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5f80 x1874531139536128/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695891 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.745624:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695866.745627:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5f80 x1874531139536128/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695891 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.745634:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.745636:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695866.745640:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695866.745643:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695866.745647:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695866.745655:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298427238 00000100:00000001:1.0:1787695866.745658:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298438457 00000100:00000001:1.0:1787695866.745660:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298427238 00000100:00000001:1.0:1787695866.745662:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298438457 00000100:00000001:1.0:1787695866.745664:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695866.745667:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.745668:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695866.745673:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695866/real 1787695866] req@ffff8b14466f5f80 x1874531139536128/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695866.745682:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695866.745684:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695866.745686:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702630. 00000400:00000200:1.0:1787695866.745688:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695866.745690:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695866.745693:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695866.745697:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5f80 x1874531139536128/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695866.745705:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5f80 x1874531139536128/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.745710:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695866.745711:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.745712:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695866.745714:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695866.745716:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5f80 x1874531139536128/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.745734:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695866.745737:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5f80 x1874531139536128/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.745742:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695866.745749:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.745750:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695866.745751:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.745752:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695866.745753:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695866.745755:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695866.745758:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695866.745759:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695866.745762:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5f80 x1874531139536128/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695866.745768:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139536128:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695866.745770:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.745771:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695866.745773:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5180 x1874531139535872/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.745777:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695866.745778:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695866.745780:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695866.745782:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695866.745784:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695866.745785:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695866.745787:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695866.745788:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:1.0:1787695866.745790:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695866.745791:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695866.745793:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5180. 00000100:00000001:1.0:1787695866.745795:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695866.745796:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.745797:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695866.745799:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6d80 x1874531139536000/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.745812:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695866.745813:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695866.745815:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695866.745818:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695866.745819:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695866.745820:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695866.745821:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695866.745822:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676000. 02000000:00000001:1.0:1787695866.745825:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695866.745825:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695866.745826:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6d80. 00000100:00000001:1.0:1787695866.745828:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695866.745829:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.745830:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695866.745832:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5f80 x1874531139536128/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.745836:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695866.745837:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695866.745838:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695866.745840:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695866.745840:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695866.745842:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695866.745843:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695866.745843:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:1.0:1787695866.745847:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695866.745848:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695866.745848:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5f80. 00000100:00000001:1.0:1787695866.745850:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695866.745850:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.745851:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.745854:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695866.745855:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.745857:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695866.745858:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695866.745859:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695866.745865:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5500 x1874531139536256/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.745870:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695866.745871:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695866.745873:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695866.745874:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695866.745875:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695866.745877:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139536256:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695866.745879:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695866.745881:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695866.745882:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695866.745884:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695866.745886:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695866.745887:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695866.745890:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695866.745892:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702630. 00000400:00000010:1.0:1787695866.745894:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695866.745896:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139536256, portal 4 00000100:00000001:1.0:1787695866.745898:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695866.745899:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930149120 : -128555779402496 : ffff8b14466f5500) 00000100:00000040:1.0:1787695866.745904:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5500 x1874531139536256/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695891 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.745908:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695866.745910:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695866.745911:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000100:00000200:1.0:1787695866.745913:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139536256, offset 0 00000400:00000200:1.0:1787695866.745916:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695866.745918:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695866.745921:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695866.745923:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695866.745925:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695866.745926:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695866.745928:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695866.745933:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [16999:16999:256:4294967295] 192.168.201.140@tcp LPNI seq info [16998:16998:8:4294967295] 00000400:00000200:1.0:1787695866.745936:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695866.745942:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695866.745945:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695866.745947:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 00000800:00000200:1.0:1787695866.745950:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695866.745953:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 3 secs later) 00000800:00000200:1.1:1787695866.745955:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695866.745957:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 3 secs later) 00000800:00000100:1.0:1787695866.745960:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695866.745961:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000400:00000200:1.0:1787695866.745964:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695866.745966:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695866.745968:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:1.0:1787695866.745970:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000100:00000001:1.0:1787695866.745972:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695866.745974:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5500 x1874531139536256/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695891 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.745980:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695866.745982:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5500 x1874531139536256/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695891 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.745986:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.745988:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695866.745990:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695866.745992:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695866.745994:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695866.745996:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298426910 00000100:00000001:1.0:1787695866.745998:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298438457 00000100:00000001:1.0:1787695866.745999:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298426910 00000100:00000001:1.0:1787695866.746001:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298438457 00000100:00000001:1.0:1787695866.746002:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695866.746008:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.746009:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695866.746013:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695866/real 1787695866] req@ffff8b14466f5500 x1874531139536256/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695866.746023:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695866.746024:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695866.746026:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702630. 00000400:00000200:1.0:1787695866.746029:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695866.746031:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695866.746033:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695866.746037:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5500 x1874531139536256/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695866.746046:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5500 x1874531139536256/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.746054:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695866.746056:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.746058:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695866.746059:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695866.746064:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5500 x1874531139536256/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.746071:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695866.746073:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5500 x1874531139536256/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.746079:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695866.746080:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.746081:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695866.746082:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.746083:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695866.746085:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695866.746086:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695866.746094:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695866.746095:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695866.746098:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5500 x1874531139536256/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695866.746104:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139536256:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695866.746106:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695866.746109:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4700 x1874531139536384/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.746113:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695866.746114:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695866.746115:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695866.746117:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695866.746118:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695866.746120:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139536384:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695866.746123:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695866.746124:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695866.746125:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695866.746126:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695866.746128:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695866.746130:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695866.746132:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695866.746134:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702630. 00000400:00000010:1.0:1787695866.746136:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695866.746139:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139536384, portal 10 00000100:00000001:1.0:1787695866.746140:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695866.746141:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930145536 : -128555779406080 : ffff8b14466f4700) 00000100:00000040:1.0:1787695866.746146:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4700 x1874531139536384/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695891 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.746151:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695866.746155:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695866.746157:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000100:00000200:1.0:1787695866.746159:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139536384, offset 0 00000400:00000200:1.0:1787695866.746162:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695866.746165:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695866.746167:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695866.746170:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695866.746171:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695866.746173:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695866.746175:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695866.746177:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17000:17000:256:4294967295] 192.168.201.140@tcp LPNI seq info [16999:16999:8:4294967295] 00000400:00000200:1.0:1787695866.746180:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695866.746184:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695866.746187:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695866.746189:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 00000800:00000200:1.0:1787695866.746191:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695866.746193:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 3 secs later) 00000800:00000200:1.1:1787695866.746196:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695866.746198:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 3 secs later) 00000800:00000100:1.0:1787695866.746201:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695866.746202:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000400:00000200:1.0:1787695866.746205:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695866.746207:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695866.746209:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:1.0:1787695866.746210:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000100:00000001:1.0:1787695866.746212:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695866.746215:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4700 x1874531139536384/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695891 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.746220:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695866.746222:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4700 x1874531139536384/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695891 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.746227:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.746228:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695866.746234:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695866.746237:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695866.746240:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695866.746242:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298427096 00000100:00000001:1.0:1787695866.746254:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298438457 00000100:00000001:1.0:1787695866.746262:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298427096 00000100:00000001:1.0:1787695866.746264:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298438457 00000100:00000001:1.0:1787695866.746267:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695866.746269:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.746271:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695866.746275:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695866/real 1787695866] req@ffff8b14466f4700 x1874531139536384/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695866.746284:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695866.746286:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695866.746288:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702630. 00000400:00000200:1.0:1787695866.746291:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695866.746293:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695866.746295:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695866.746299:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4700 x1874531139536384/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695866.746306:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4700 x1874531139536384/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.746311:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695866.746312:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.746319:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695866.746320:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695866.746323:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4700 x1874531139536384/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.746330:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695866.746333:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4700 x1874531139536384/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.746344:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695866.746346:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.746347:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695866.746348:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.746349:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695866.746350:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695866.746352:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695866.746354:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695866.746355:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695866.746358:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4700 x1874531139536384/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695866.746380:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139536384:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695866.746388:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695866.746393:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4000 x1874531139536512/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.746398:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695866.746399:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695866.746401:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695866.746402:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695866.746403:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695866.746405:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139536512:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695866.746408:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695866.746410:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695866.746411:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695866.746412:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695866.746414:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695866.746415:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695866.746421:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695866.746423:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702630. 00000400:00000010:1.0:1787695866.746426:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695866.746428:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139536512, portal 10 00000100:00000001:1.0:1787695866.746430:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695866.746431:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930143744 : -128555779407872 : ffff8b14466f4000) 00000100:00000040:1.0:1787695866.746436:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4000 x1874531139536512/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695891 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.746440:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695866.746442:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695866.746443:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000100:00000200:1.0:1787695866.746445:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139536512, offset 0 00000400:00000200:1.0:1787695866.746450:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695866.746454:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695866.746458:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695866.746461:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695866.746463:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695866.746466:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695866.746468:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695866.746472:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17001:17001:256:4294967295] 192.168.201.140@tcp LPNI seq info [17000:17000:8:4294967295] 00000400:00000200:1.0:1787695866.746476:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695866.746483:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695866.746488:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695866.746492:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 00000800:00000200:1.0:1787695866.746496:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695866.746499:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 3 secs later) 00000800:00000200:1.1:1787695866.746504:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695866.746507:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 32, 3 secs later) 00000800:00000100:1.0:1787695866.746510:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695866.746511:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000400:00000200:1.0:1787695866.746514:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695866.746516:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695866.746519:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:1.0:1787695866.746524:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000100:00000001:1.0:1787695866.746527:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695866.746529:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4000 x1874531139536512/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695891 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.746536:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695866.746538:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4000 x1874531139536512/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695891 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.746542:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.746543:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695866.746546:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695866.746548:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695866.746551:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695866.746552:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298427075 00000100:00000001:1.0:1787695866.746554:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298438457 00000100:00000001:1.0:1787695866.746556:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298427075 00000100:00000001:1.0:1787695866.746557:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298438457 00000100:00000001:1.0:1787695866.746559:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695866.746560:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.746561:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695866.746563:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695866/real 1787695866] req@ffff8b14466f4000 x1874531139536512/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695866.746569:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695866.746569:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695866.746571:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702630. 00000400:00000200:1.0:1787695866.746573:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695866.746574:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695866.746575:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695866.746577:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4000 x1874531139536512/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695866.746583:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4000 x1874531139536512/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.746590:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695866.746591:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.746593:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695866.746594:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695866.746596:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4000 x1874531139536512/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.746600:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695866.746603:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4000 x1874531139536512/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.746608:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695866.746609:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.746610:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695866.746611:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.746612:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695866.746613:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695866.746615:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695866.746617:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695866.746618:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695866.746620:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4000 x1874531139536512/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695866.746625:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139536512:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695866.746627:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.746629:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695866.746631:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5500 x1874531139536256/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.746636:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695866.746639:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695866.746641:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695866.746643:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695866.746644:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695866.746646:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695866.746647:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695866.746648:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675400. 02000000:00000001:1.0:1787695866.746651:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695866.746652:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695866.746653:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5500. 00000100:00000001:1.0:1787695866.746655:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695866.746656:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.746657:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695866.746660:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4700 x1874531139536384/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.746664:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695866.746664:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695866.746665:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695866.746666:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695866.746667:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695866.746668:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695866.746669:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695866.746670:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:1.0:1787695866.746671:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695866.746672:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695866.746672:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4700. 00000100:00000001:1.0:1787695866.746674:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695866.746675:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.746676:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695866.746678:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4000 x1874531139536512/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695891 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695866.746682:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695866.746683:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695866.746684:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695866.746685:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695866.746686:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695866.746689:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695866.746691:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695866.746692:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:1.0:1787695866.746694:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695866.746695:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695866.746696:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4000. 00000100:00000001:1.0:1787695866.746697:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695866.746698:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.746699:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695866.746702:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695866.746703:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695866.746704:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695866.746705:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695866.806484:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695866.806493:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695866.806514:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695866.806520:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695866.806524:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695866.806528:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695866.806531:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695866.806533:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695866.806536:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695867.446466:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695867.446471:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695867.446475:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695867.446476:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695867.446479:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695867.446480:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695867.446483:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695867.446484:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695867.446486:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695867.446487:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695867.446490:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695867.446491:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695867.446548:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695867.446552:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695867.446554:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695867.830451:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695867.830463:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695867.830466:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695867.830474:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695867.830479:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695867.830484:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695867.830487:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695867.830491:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695867.830494:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695868.470495:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695868.470500:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695868.470505:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695868.470506:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695868.470508:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695868.470510:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695868.470512:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695868.470513:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695868.470516:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695868.470517:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695868.470519:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695868.470520:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695868.470612:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695868.470616:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695868.470619:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695868.854461:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695868.854468:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695868.854470:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695868.854474:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695868.854477:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695868.854480:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695868.854482:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695868.854483:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695868.854485:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695869.494457:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695869.494483:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695869.494490:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695869.494492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695869.494496:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695869.494497:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695869.494501:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695869.494503:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695869.494506:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695869.494508:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695869.494511:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695869.494513:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695869.494593:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695869.494597:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695869.494599:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695869.878413:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695869.878420:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695869.878422:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695869.878426:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695869.878429:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695869.878432:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695869.878434:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695869.878436:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695869.878438:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695870.518432:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695870.518440:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695870.518446:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695870.518447:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695870.518459:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695870.518461:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695870.518466:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695870.518468:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695870.518471:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695870.518473:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695870.518476:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695870.518477:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695870.518546:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695870.518551:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695870.518554:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695870.902518:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695870.902552:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695870.902554:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695870.902563:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695870.902568:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695870.902573:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695870.902576:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695870.902580:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695870.902584:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695871.542449:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695871.542455:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695871.542459:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695871.542460:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695871.542463:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695871.542464:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695871.542467:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695871.542468:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695871.542470:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695871.542472:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695871.542475:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695871.542479:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695871.542481:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695871.542482:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695871.542484:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695871.542504:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695871.542506:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695871.542508:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695871.542511:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695871.542511:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695871.542558:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695871.542562:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695871.542564:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787695871.862553:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695871.862583:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787695871.862587:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695871.862592:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695871.862597:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695871.862606:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3759 00000400:00000200:3.0:1787695871.862619:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695871.862628:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695871.862644:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695871.862649:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695871.862654:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695871.862657:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695871.862660:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695871.862665:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695871.862668:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:3.0:1787695871.862671:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695871.862676:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787695871.862678:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787695871.862681:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787695871.862685:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695871.862687:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695871.862688:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695871.862690:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695871.862694:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787695871.862715:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695871.862718:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695871.862722:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695871.862726:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695871.862729:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.862740:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787695871.862744:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695871.862752:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4a80. 00000020:00000040:3.0:1787695871.862757:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787695871.862762:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695871.862765:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.862769:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695871.862772:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695871.862779:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695871.862789:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675800. 00000100:00000001:3.0:1787695871.862797:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695871.862808:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4a80 x1874531139536640/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695871.862821:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695871.862827:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695871.862833:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ad0d67] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4a80 x1874531139536640/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695871.862862:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695871.862865:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695871.862867:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695871.862869:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695871.862870:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695871.862873:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695871.862875:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3759 00000400:00000200:3.0:1787695871.862879:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695871.862881:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695871.862883:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695871.862892:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695871.862894:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695871.862896:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695871.862898:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695871.862901:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695871.862902:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:3.0:1787695871.862903:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695871.862905:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:3.0:1787695871.862907:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:3.0:1787695871.862908:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:3.0:1787695871.862910:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695871.862912:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695871.862913:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695871.862914:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695871.862916:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695871.862919:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:3.0:1787695871.862922:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695871.862923:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695871.862927:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695871.862928:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787695871.862930:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695871.862936:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1787695871.862937:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4a80 x1874531139536640/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695871.862940:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.862945:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695871.862952:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1787695871.862952:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787695871.862956:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787695871.862956:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.862960:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695871.862961:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:3.0:1787695871.862963:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6a00. 02000000:00000001:1.0:1787695871.862965:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1787695871.862967:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:1.0:1787695871.862969:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695871.862971:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695871.862973:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695871.862974:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139536640:192.168.201.140@tcp:250:kworker.0 00000100:00000001:3.0:1787695871.862976:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695871.862980:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787695871.862982:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787695871.862984:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695871.862987:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695871.862990:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695871.862992:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1787695871.862994:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 02000000:00000010:1.0:1787695871.862998:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546000. 00000100:00000001:3.0:1787695871.863006:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695871.863007:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695871.863017:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00100000:3.0:1787695871.863018:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6a00 x1874531139536768/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1787695871.863022:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702630. 00000400:00000010:1.0:1787695871.863031:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000001:3.0:1787695871.863032:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695871.863036:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695871.863040:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002ad47e8f] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6a00 x1874531139536768/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695871.863042:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139536640, portal 25 00000100:00000001:1.0:1787695871.863045:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695871.863047:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930146432 : -128555779405184 : ffff8b14466f4a80) 00000100:00000040:1.0:1787695871.863054:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4a80 x1874531139536640/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695896 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695871.863060:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695871.863066:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695871.863067:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695871.863073:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00080000:3.0:1787695871.863073:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695871.863077:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:1.0:1787695871.863079:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46110. 00000100:00080000:3.0:1787695871.863080:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1787695871.863084:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139536640, offset 0 00000100:00000001:3.0:1787695871.863085:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695871.863089:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3759 00000400:00000200:3.0:1787695871.863098:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695871.863103:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:3.0:1787695871.863107:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695871.863111:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1787695871.863116:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695871.863117:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695871.863123:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695871.863126:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695871.863129:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:3.0:1787695871.863130:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695871.863137:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695871.863139:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000400:00000200:1.0:1787695871.863141:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1787695871.863142:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695871.863146:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:3.0:1787695871.863148:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000400:00000200:1.0:1787695871.863150:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000020:00000001:3.0:1787695871.863151:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:3.0:1787695871.863157:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695871.863159:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1787695871.863160:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695871.863165:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000040:3.0:1787695871.863171:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695871.863176:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695871.863177:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17002:17002:256:4294967295] 192.168.201.140@tcp LPNI seq info [17001:17001:8:4294967295] 00000020:00000040:3.0:1787695871.863179:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:3.0:1787695871.863185:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695871.863188:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695871.863190:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787695871.863191:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695871.863195:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695871.863197:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.863199:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695871.863201:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695871.863204:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.863208:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695871.863212:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7b80. 00000400:00000200:1.0:1787695871.863213:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000020:00000040:3.0:1787695871.863215:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695871.863220:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695871.863221:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.863223:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695871.863224:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695871.863225:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695871.863228:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 00000100:00000001:3.0:1787695871.863231:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695871.863235:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00100000:3.0:1787695871.863237:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7b80 x1874531139536896/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:1.0:1787695871.863243:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817300. 00000100:00000001:3.0:1787695871.863245:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695871.863248:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695871.863252:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b87e1da] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7b80 x1874531139536896/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1787695871.863255:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695871.863265:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787695871.863266:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695871.863270:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695871.863274:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787695871.863275:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695871.863280:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695871.863283:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695871.863286:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695871.863287:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3759 00000400:00000200:3.0:1787695871.863293:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695871.863295:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695871.863299:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695871.863301:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298443576 00000100:00080000:3.0:1787695871.863302:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695871.863307:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695871.863309:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695871.863311:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695871.863312:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298438457 00000100:00000001:3.0:1787695871.863314:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695871.863318:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695871.863319:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000100:00000001:1.0:1787695871.863321:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298443576 00000020:00000001:3.0:1787695871.863321:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695871.863323:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:3.0:1787695871.863326:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:1.0:1787695871.863328:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298438457 00000020:00000001:3.0:1787695871.863329:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:1.0:1787695871.863333:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1787695871.863334:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695871.863340:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.863340:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695871.863345:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695871.863346:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.863347:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695871.863350:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1787695871.863351:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:3.0:1787695871.863356:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695871.863359:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695871.863360:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1787695871.863392:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695871.863398:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695871.863401:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695871.863404:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1787695871.863405:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695871.863408:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:3.0:1787695871.863411:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787695871.863412:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695871.863415:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00000001:3.0:1787695871.863417:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.863419:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695871.863421:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6a00 x1874531139536768/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1787695871.863431:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5880. 00000100:00000001:1.0:1787695871.863432:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000040:3.0:1787695871.863435:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:3.0:1787695871.863438:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695871.863439:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695871.863442:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.863442:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695871.863446:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695871.863447:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787695871.863449:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695871.863450:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695871.863454:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695871.863455:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675400. 00000100:00100000:1.0:1787695871.863457:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139536768:192.168.201.140@tcp:38:kworker.0 00000100:00000001:3.0:1787695871.863460:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695871.863461:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695871.863465:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695871.863467:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695871.863469:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5880 x1874531139537024/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787695871.863470:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695871.863474:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 00000100:00000001:3.0:1787695871.863478:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695871.863480:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695871.863483:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000827e7dbe] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5880 x1874531139537024/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695871.863488:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695871.863497:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695871.863497:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695871.863501:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:1.0:1787695871.863502:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702c08. 00000100:00080000:3.0:1787695871.863508:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000010:1.0:1787695871.863510:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46e58. 00000100:00000001:3.0:1787695871.863510:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695871.863513:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1787695871.863515:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139536768, portal 10 00000100:00000001:3.0:1787695871.863516:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695871.863520:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787695871.863521:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3759 00000100:00000001:1.0:1787695871.863522:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930154496 : -128555779397120 : ffff8b14466f6a00) 00000400:00000200:3.0:1787695871.863527:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:1.0:1787695871.863534:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6a00 x1874531139536768/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695896 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695871.863534:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695871.863538:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695871.863544:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695871.863548:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787695871.863548:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695871.863555:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695871.863555:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000010:1.0:1787695871.863561:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f466e8. 00000100:00000001:3.0:1787695871.863561:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000200:1.0:1787695871.863567:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139536768, offset 0 00000020:00000001:3.0:1787695871.863567:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:1.0:1787695871.863573:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000020:00000040:3.0:1787695871.863577:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000400:00000200:1.0:1787695871.863578:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:3.0:1787695871.863579:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695871.863583:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000400:00000200:1.0:1787695871.863584:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000020:00000001:3.0:1787695871.863586:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000400:00000200:1.0:1787695871.863590:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1787695871.863591:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000400:00000200:1.0:1787695871.863594:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000040:3.0:1787695871.863597:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695871.863600:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1787695871.863601:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695871.863607:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000040:3.0:1787695871.863608:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695871.863612:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695871.863617:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17003:17003:255:4294967295] 192.168.201.140@tcp LPNI seq info [17002:17002:7:4294967295] 00000020:00000040:3.0:1787695871.863617:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:3.0:1787695871.863624:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:1.0:1787695871.863626:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695871.863627:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695871.863630:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695871.863647:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1787695871.863653:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 02000000:00000001:3.0:1787695871.863654:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.863657:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695871.863659:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787695871.863665:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.863669:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695871.863674:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000010:3.0:1787695871.863677:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6680. 00000800:00000010:1.0:1787695871.863681:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817400. 00000020:00000040:3.0:1787695871.863682:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:3.0:1787695871.863688:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000200:1.0:1787695871.863690:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:3.0:1787695871.863691:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.863696:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695871.863701:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:1.1:1787695871.863701:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:3.0:1787695871.863705:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695871.863707:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695871.863712:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 00000100:00000040:1.0:1787695871.863714:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000100:00000001:3.0:1787695871.863720:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695871.863721:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695871.863727:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298433478 00000100:00100000:3.0:1787695871.863731:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6680 x1874531139537152/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695871.863732:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298443576 00000100:00000001:1.0:1787695871.863738:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298433478 00000100:00000001:1.0:1787695871.863744:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298443576 00000100:00000001:1.0:1787695871.863749:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695871.863751:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695871.863753:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.863756:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695871.863759:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695871.863767:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7b80 x1874531139536896/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695871.863772:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f6d2abf9] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6680 x1874531139537152/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695871.863786:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695871.863790:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.863793:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695871.863795:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1787695871.863797:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1787695871.863799:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1787695871.863801:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 02000000:00000001:1.0:1787695871.863806:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.863806:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695871.863810:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787695871.863814:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139536896:192.168.201.140@tcp:38:kworker.0 00000100:00000001:3.0:1787695871.863816:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695871.863822:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1787695871.863823:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3759 02000000:00000001:1.0:1787695871.863826:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695871.863830:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1787695871.863831:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:1.0:1787695871.863835:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:3.0:1787695871.863839:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 02000000:00000010:1.0:1787695871.863843:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 00000100:00080000:3.0:1787695871.863845:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1787695871.863848:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695871.863853:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695871.863858:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695871.863860:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1787695871.863863:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547022c0. 00000100:00000040:3.0:1787695871.863865:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000010:1.0:1787695871.863871:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000001:3.0:1787695871.863871:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695871.863874:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695871.863876:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:3.0:1787695871.863879:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000200:1.0:1787695871.863884:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139536896, portal 10 00000020:00000040:3.0:1787695871.863884:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000020:00000001:3.0:1787695871.863889:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:1.0:1787695871.863890:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695871.863894:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930158976 : -128555779392640 : ffff8b14466f7b80) 00000020:00000001:3.0:1787695871.863895:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000040:3.0:1787695871.863901:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695871.863904:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695871.863907:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7b80 x1874531139536896/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695896 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695871.863910:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695871.863915:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695871.863921:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00000001:1.0:1787695871.863925:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787695871.863927:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000040:1.0:1787695871.863933:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695871.863934:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695871.863939:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46770. 00000100:00000200:1.0:1787695871.863943:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139536896, offset 0 02000000:00000001:3.0:1787695871.863946:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1787695871.863952:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695871.863952:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695871.863957:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695871.863962:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695871.863968:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:1.0:1787695871.863969:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000008:00100000:3.0:1787695871.863974:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1787695871.863977:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000008:00000001:3.0:1787695871.863980:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695871.863983:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:3.0:1787695871.863985:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695871.863988:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000010:3.0:1787695871.863993:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5c00. 00000400:00000200:1.0:1787695871.863994:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000020:00000040:3.0:1787695871.863999:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000400:00000200:1.0:1787695871.864002:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17004:17004:254:4294967295] 192.168.201.140@tcp LPNI seq info [17003:17003:6:4294967295] 02000000:00000001:3.0:1787695871.864005:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695871.864008:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695871.864013:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695871.864014:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695871.864019:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695871.864022:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695871.864028:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 02000000:00000010:3.0:1787695871.864030:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676000. 00000800:00000200:1.0:1787695871.864037:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695871.864038:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1787695871.864045:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817100. 00000100:00100000:3.0:1787695871.864048:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5c00 x1874531139537280/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1787695871.864055:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695871.864065:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787695871.864067:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695871.864072:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.864072:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695871.864080:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000100:00000040:3.0:1787695871.864082:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005bfbcee1] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5c00 x1874531139537280/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695871.864088:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695871.864093:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298432326 00000100:00000001:1.0:1787695871.864099:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298443576 00000100:00000001:3.0:1787695871.864102:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695871.864104:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298432326 00000100:00080000:3.0:1787695871.864108:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695871.864110:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298443576 00000100:00000001:1.0:1787695871.864116:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1787695871.864117:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695871.864119:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695871.864121:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695871.864126:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695871.864128:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695871.864132:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3759 00000100:00000001:1.0:1787695871.864134:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695871.864139:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787695871.864141:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695871.864145:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:3.0:1787695871.864149:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695871.864150:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00080000:3.0:1787695871.864154:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695871.864156:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695871.864158:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695871.864160:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695871.864167:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5880 x1874531139537024/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695871.864169:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695871.864175:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695871.864181:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695871.864186:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695871.864188:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695871.864192:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695871.864192:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695871.864195:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 02000000:00000001:1.0:1787695871.864198:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1787695871.864199:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1787695871.864203:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1787695871.864204:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 02000000:00000001:1.0:1787695871.864210:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695871.864211:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000020:00000001:3.0:1787695871.864217:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00100000:1.0:1787695871.864219:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139537024:192.168.201.140@tcp:8:kworker.0 00000100:00000040:3.0:1787695871.864224:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695871.864227:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787695871.864228:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787695871.864232:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1787695871.864233:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695871.864236:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.864238:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:1.0:1787695871.864241:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:3.0:1787695871.864244:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 02000000:00000010:1.0:1787695871.864247:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 00000100:00080000:3.0:1787695871.864250:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000001:1.0:1787695871.864253:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.864256:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695871.864260:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1787695871.864262:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787695871.864265:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1787695871.864268:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702688. 02000000:00000001:3.0:1787695871.864270:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.864274:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:1.0:1787695871.864276:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f464c8. 00000008:00100000:3.0:1787695871.864277:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000200:1.0:1787695871.864282:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139537024, portal 4 00000008:00000001:3.0:1787695871.864283:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695871.864289:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787695871.864290:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695871.864293:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150016 : -128555779401600 : ffff8b14466f5880) 00000100:00000010:3.0:1787695871.864298:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7800. 00000020:00000040:3.0:1787695871.864303:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000040:1.0:1787695871.864304:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5880 x1874531139537024/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695896 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695871.864308:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695871.864311:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695871.864319:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695871.864322:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695871.864326:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46088. 00000100:00000200:1.0:1787695871.864329:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139537024, offset 0 00000400:00000200:1.0:1787695871.864340:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695871.864345:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695871.864349:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695871.864352:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695871.864355:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695871.864358:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695871.864360:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695871.864388:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17005:17005:253:4294967295] 192.168.201.140@tcp LPNI seq info [17004:17004:5:4294967295] 00000400:00000200:1.0:1787695871.864395:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695871.864404:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695871.864409:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695871.864413:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817700. 00000800:00000200:1.0:1787695871.864418:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695871.864422:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:1.0:1787695871.864426:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695871.864430:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695871.864434:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695871.864437:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298432357 00000100:00000001:1.0:1787695871.864439:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298443576 00000100:00000001:1.0:1787695871.864444:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298432357 00000100:00000001:3.0:1787695871.864445:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787695871.864449:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298443576 02000000:00000001:3.0:1787695871.864450:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695871.864454:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695871.864455:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695871.864460:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695871.864463:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000010:3.0:1787695871.864463:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 00000100:00000040:1.0:1787695871.864469:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6680 x1874531139537152/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695871.864477:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695871.864481:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695871.864484:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695871.864489:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:3.0:1787695871.864489:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7800 x1874531139537408/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787695871.864493:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695871.864498:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.864503:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695871.864506:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695871.864514:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139537152:192.168.201.140@tcp:8:kworker.0 00000100:00000040:3.0:1787695871.864515:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7c57981] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7800 x1874531139537408/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695871.864525:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695871.864529:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695871.864533:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.864533:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695871.864539:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:3.0:1787695871.864540:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000010:1.0:1787695871.864544:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545000. 00000100:00000040:3.0:1787695871.864545:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 02000000:00000001:1.0:1787695871.864550:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695871.864552:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787695871.864558:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1787695871.864559:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695871.864565:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000100:00000040:3.0:1787695871.864568:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3769) 00000400:00000010:1.0:1787695871.864570:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46220. 00000100:00000200:1.0:1787695871.864576:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139537152, portal 4 00000100:00000001:1.0:1787695871.864579:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695871.864581:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930153600 : -128555779398016 : ffff8b14466f6680) 00000100:00000040:1.0:1787695871.864590:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6680 x1874531139537152/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695896 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695871.864598:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695871.864601:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695871.864605:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46550. 00000100:00000200:1.0:1787695871.864608:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139537152, offset 0 00000400:00000200:1.0:1787695871.864614:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695871.864619:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695871.864624:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695871.864627:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695871.864630:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695871.864646:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695871.864648:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695871.864652:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17006:17006:252:4294967295] 192.168.201.140@tcp LPNI seq info [17005:17005:4:4294967295] 00000400:00000200:1.0:1787695871.864656:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695871.864663:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695871.864670:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695871.864674:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1449817e00. 00000800:00000200:1.0:1787695871.864686:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695871.864691:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:1.0:1787695871.864695:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695871.864698:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695871.864701:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695871.864704:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298432029 00000100:00000001:1.0:1787695871.864707:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298443576 00000100:00000001:1.0:1787695871.864709:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298432029 00000100:00000001:1.0:1787695871.864711:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298443576 00000100:00000001:1.0:1787695871.864714:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695871.864716:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695871.864718:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695871.864723:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5c00 x1874531139537280/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695871.864731:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695871.864733:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695871.864735:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695871.864737:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695871.864739:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695871.864743:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139537280:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695871.864747:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695871.864749:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695871.864751:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695871.864753:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695871.864757:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544c00. 02000000:00000001:1.0:1787695871.864759:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695871.864763:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695871.864766:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547027e8. 00000400:00000010:1.0:1787695871.864770:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46bb0. 00000100:00000200:1.0:1787695871.864773:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139537280, portal 10 00000100:00000001:1.0:1787695871.864776:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695871.864777:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150912 : -128555779400704 : ffff8b14466f5c00) 00000100:00000040:1.0:1787695871.864792:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5c00 x1874531139537280/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695896 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695871.864799:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695871.864801:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695871.864804:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46660. 00000100:00000200:1.0:1787695871.864807:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139537280, offset 0 00000400:00000200:1.0:1787695871.864812:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695871.864816:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695871.864820:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695871.864823:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695871.864826:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695871.864829:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695871.864831:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695871.864835:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17007:17007:251:4294967295] 192.168.201.140@tcp LPNI seq info [17006:17006:3:4294967295] 00000400:00000200:1.0:1787695871.864839:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695871.864846:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695871.864851:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695871.864856:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b144a720700. 00000800:00000200:1.0:1787695871.864861:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695871.864865:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:1.0:1787695871.864876:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695871.864880:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695871.864884:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695871.864886:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298432215 00000100:00000001:1.0:1787695871.864888:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298443576 00000100:00000001:1.0:1787695871.864890:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298432215 00000100:00000001:1.0:1787695871.864892:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298443576 00000100:00000001:1.0:1787695871.864895:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695871.864897:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695871.864900:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695871.864909:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695871.864914:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695871.864916:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787695871.864918:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695871.864920:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695871.864921:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695871.864926:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7800 x1874531139537408/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695871.864933:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695871.864935:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695871.864937:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695871.864939:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695871.864941:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695871.864944:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139537408:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695871.864947:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695871.864949:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695871.864951:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695871.864952:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695871.864956:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695871.864958:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695871.864961:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695871.864964:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695871.864968:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695871.864971:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139537408, portal 10 00000100:00000001:1.0:1787695871.864973:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695871.864974:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930158080 : -128555779393536 : ffff8b14466f7800) 00000100:00000040:1.0:1787695871.864980:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7800 x1874531139537408/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695896 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695871.864987:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695871.864989:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695871.864992:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46440. 00000100:00000200:1.0:1787695871.864994:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139537408, offset 0 00000400:00000200:1.0:1787695871.864998:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695871.865002:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695871.865011:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695871.865015:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695871.865017:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695871.865020:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695871.865023:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695871.865026:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17008:17008:250:4294967295] 192.168.201.140@tcp LPNI seq info [17007:17007:2:4294967295] 00000400:00000200:1.0:1787695871.865031:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695871.865038:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695871.865043:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695871.865048:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981300. 00000800:00000200:1.0:1787695871.865052:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695871.865056:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:1.0:1787695871.865060:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695871.865063:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695871.865067:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695871.865069:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298432193 00000100:00000001:1.0:1787695871.865072:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298443576 00000100:00000001:1.0:1787695871.865074:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298432193 00000100:00000001:1.0:1787695871.865076:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298443576 00000100:00000001:1.0:1787695871.865079:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695871.865081:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695871.865085:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695871.865087:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695871.865090:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695871.865092:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787695871.865094:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695871.865096:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695871.865098:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695871.865100:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695871.926502:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695871.926511:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695871.926514:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695871.926519:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695871.926522:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695871.926527:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695871.926529:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695871.926532:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695871.926535:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695872.566470:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695872.566476:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695872.566480:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695872.566480:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695872.566499:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695872.566500:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695872.566503:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695872.566504:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695872.566506:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695872.566507:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695872.566509:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695872.566510:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695872.566598:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695872.566602:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695872.566604:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695872.950477:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695872.950484:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695872.950486:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695872.950490:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695872.950493:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695872.950496:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695872.950514:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695872.950516:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695872.950518:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695873.590486:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695873.590497:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695873.590500:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695873.590503:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695873.590509:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695873.590511:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695873.590514:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695873.590515:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695873.590519:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695873.590520:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695873.590521:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695873.590523:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695873.590524:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695873.590525:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695873.590526:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695873.590527:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695873.590531:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695873.590532:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695873.590535:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695873.590536:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695873.590539:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695873.590539:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695873.590542:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695873.590543:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695873.590622:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695873.590626:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695873.590629:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695873.974521:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695873.974533:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695873.974536:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695873.974543:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695873.974548:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695873.974554:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695873.974557:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695873.974561:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695873.974564:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695874.614500:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695874.614507:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695874.614514:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695874.614516:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695874.614520:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695874.614521:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695874.614542:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695874.614543:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695874.614545:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695874.614546:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695874.614548:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695874.614549:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695874.614648:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695874.614654:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695874.614658:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787695874.934708:0:2393:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.140:988 00000400:00000100:2.0:1787695874.934773:0:2393:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.140@tcp at host 192.168.201.140 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:2.0:1787695874.934784:0:2393:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695874.934788:0:2393:0:(router.c:1791:lnet_notify()) 192.168.201.40@tcp notifying 192.168.201.140@tcp: down 00000400:00000200:2.0:1787695874.934795:0:2393:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695874.934810:0:2393:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:2.0:1787695874.934821:0:2393:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:2.0:1787695874.934825:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817300. 00000400:00000200:2.0:1787695874.934835:0:2393:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1787695874.934840:0:2393:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1787695874.934849:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46110 00000400:00000010:2.0:1787695874.934852:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46110. 00000100:00000001:2.0:1787695874.934859:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695874.934871:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f4a80 x1874531139536640/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695896 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695874.934902:0:2393:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695874.934908:0:2393:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4a80 x1874531139536640/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695896 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695874.934918:0:2393:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695874.934921:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695874.934964:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000100:2.0:1787695874.934964:0:2393:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:2.0:1787695874.934969:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817400. 00000100:00000001:1.0:1787695874.934975:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000200:2.0:1787695874.934977:0:2393:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1787695874.934985:0:2393:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000100:00000001:1.0:1787695874.934986:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000400:00000200:2.0:1787695874.934990:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f466e8 00000400:00000010:2.0:1787695874.934995:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f466e8. 00000100:00000001:2.0:1787695874.934998:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:1.0:1787695874.935006:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695871/real 1787695874] req@ffff8b14466f4a80 x1874531139536640/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695874.935007:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f6a00 x1874531139536768/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695896 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695874.935058:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695874.935061:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695874.935067:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702630. 00000100:00000001:2.0:1787695874.935067:0:2393:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1787695874.935077:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000100:00000040:2.0:1787695874.935078:0:2393:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6a00 x1874531139536768/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695896 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1787695874.935083:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695874.935093:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787695874.935095:0:2393:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695874.935099:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000200:1.0:1787695874.935104:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4a80 x1874531139536640/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1787695874.935117:0:2393:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:2.0:1787695874.935128:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817100. 00000400:00000200:2.0:1787695874.935136:0:2393:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000200:1.0:1787695874.935145:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4a80 x1874531139536640/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787695874.935146:0:2393:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1787695874.935158:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46770 00000400:00000010:2.0:1787695874.935163:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46770. 00000100:00000001:1.0:1787695874.935169:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695874.935171:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787695874.935176:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:2.0:1787695874.935180:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f7b80 x1874531139536896/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695896 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.935183:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695874.935189:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695874.935202:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4a80 x1874531139536640/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695874.935205:0:2393:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695874.935214:0:2393:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7b80 x1874531139536896/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695896 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.935228:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695874.935239:0:2393:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695874.935244:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1787695874.935247:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4a80 x1874531139536640/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1787695874.935260:0:2393:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:2.0:1787695874.935269:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817700. 00000100:00000001:1.0:1787695874.935273:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:2.0:1787695874.935277:0:2393:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:1.0:1787695874.935281:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1787695874.935294:0:2393:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000100:00000001:1.0:1787695874.935303:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000400:00000200:2.0:1787695874.935306:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46088 00000100:00000001:1.0:1787695874.935308:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1787695874.935312:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46088. 00000100:00000001:1.0:1787695874.935315:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695874.935321:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695874.935321:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:1.0:1787695874.935330:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000200:2.0:1787695874.935333:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f5880 x1874531139537024/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695896 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695874.935340:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695874.935348:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695874.935397:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4a80 x1874531139536640/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695874.935410:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139536640:192.168.201.140@tcp:250:kworker.0 00000100:00000001:2.0:1787695874.935418:0:2393:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787695874.935419:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695874.935426:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695874.935430:0:2393:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5880 x1874531139537024/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695896 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695874.935437:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4a80 x1874531139536640/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695874.935460:0:2393:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695874.935467:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695874.935470:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695874.935476:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000800:00000100:2.0:1787695874.935484:0:2393:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 02000000:00000010:1.0:1787695874.935485:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546000. 00000800:00000010:2.0:1787695874.935494:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1449817e00. 02000000:00000001:1.0:1787695874.935498:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:2.0:1787695874.935502:0:2393:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000020:00000001:1.0:1787695874.935504:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695874.935508:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000400:00000200:2.0:1787695874.935513:0:2393:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000020:00000001:1.0:1787695874.935514:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695874.935521:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675800. 00000400:00000200:2.0:1787695874.935522:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46550 00000400:00000010:2.0:1787695874.935529:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46550. 02000000:00000001:1.0:1787695874.935534:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1787695874.935536:0:2393:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1787695874.935539:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695874.935544:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4a80. 00000100:00000200:2.0:1787695874.935545:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f6680 x1874531139537152/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695896 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.935550:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695874.935554:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695874.935566:0:2393:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787695874.935571:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695874.935574:0:2393:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6680 x1874531139537152/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695896 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.935577:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695874.935598:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695874.935601:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787695874.935607:0:2393:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695874.935612:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:1.0:1787695874.935615:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695871/real 1787695874] req@ffff8b14466f5880 x1874531139537024/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1787695874.935628:0:2393:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:2.0:1787695874.935636:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b144a720700. 00000400:00000200:2.0:1787695874.935644:0:2393:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:2.0:1787695874.935657:0:2393:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1787695874.935670:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46660 00000400:00000010:2.0:1787695874.935675:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46660. 00000400:00000001:1.0:1787695874.935682:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695874.935684:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695874.935687:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702688. 00000400:00000200:1.0:1787695874.935691:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f464c8 00000400:00000010:1.0:1787695874.935696:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f464c8. 00000100:00000001:2.0:1787695874.935698:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787695874.935703:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695874.935710:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f5c00 x1874531139537280/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695896 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695874.935715:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5880 x1874531139537024/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695874.935758:0:2393:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000200:1.0:1787695874.935760:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5880 x1874531139537024/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695874.935769:0:2393:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5c00 x1874531139537280/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695896 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.935785:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695874.935790:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695874.935790:0:2393:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695874.935798:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695874.935799:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695874.935803:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695874.935812:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5880 x1874531139537024/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1787695874.935815:0:2393:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:2.0:1787695874.935823:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981300. 00000400:00000200:2.0:1787695874.935833:0:2393:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:1.0:1787695874.935842:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1787695874.935848:0:2393:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:2.0:1787695874.935854:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46440 00000400:00000010:2.0:1787695874.935857:0:2393:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46440. 00000100:00000001:2.0:1787695874.935860:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:1.0:1787695874.935865:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5880 x1874531139537024/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695874.935868:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f7800 x1874531139537408/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695896 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695874.935902:0:2393:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787695874.935906:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:2.0:1787695874.935909:0:2393:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7800 x1874531139537408/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695896 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.935914:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.935921:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695874.935925:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.935932:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695874.935936:0:2393:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695874.935937:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695874.935944:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:1.0:1787695874.935947:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695874.935955:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695874.935958:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695874.935973:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5880 x1874531139537024/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695874.935982:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139537024:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695874.935985:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695874.935988:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695871/real 1787695874] req@ffff8b14466f6680 x1874531139537152/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695874.935995:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695874.935996:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695874.935997:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695874.936000:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46220 00000400:00000010:1.0:1787695874.936001:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46220. 00000100:00000001:1.0:1787695874.936004:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695874.936006:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6680 x1874531139537152/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695874.936011:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6680 x1874531139537152/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.936016:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695874.936017:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936018:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695874.936019:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695874.936021:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6680 x1874531139537152/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.936025:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695874.936027:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6680 x1874531139537152/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.936032:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695874.936033:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936038:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695874.936039:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936040:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695874.936042:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695874.936044:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695874.936046:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695874.936047:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695874.936050:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6680 x1874531139537152/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695874.936055:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139537152:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695874.936058:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695874.936061:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695871/real 1787695874] req@ffff8b14466f5c00 x1874531139537280/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695874.936066:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695874.936067:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695874.936069:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547027e8. 00000400:00000200:1.0:1787695874.936070:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46bb0 00000400:00000010:1.0:1787695874.936071:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46bb0. 00000100:00000001:1.0:1787695874.936072:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695874.936074:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5c00 x1874531139537280/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695874.936079:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5c00 x1874531139537280/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.936083:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695874.936083:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936084:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695874.936085:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695874.936087:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5c00 x1874531139537280/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.936098:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695874.936101:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5c00 x1874531139537280/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.936105:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695874.936107:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936108:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695874.936108:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936110:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695874.936111:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695874.936112:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695874.936114:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695874.936115:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695874.936118:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5c00 x1874531139537280/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695874.936122:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139537280:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695874.936124:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695874.936127:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695871/real 1787695874] req@ffff8b14466f6a00 x1874531139536768/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695874.936132:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695874.936133:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695874.936134:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702c08. 00000400:00000200:1.0:1787695874.936135:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46e58 00000400:00000010:1.0:1787695874.936136:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46e58. 00000100:00000001:1.0:1787695874.936137:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695874.936139:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6a00 x1874531139536768/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695874.936153:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6a00 x1874531139536768/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.936157:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695874.936158:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936159:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695874.936160:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695874.936162:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6a00 x1874531139536768/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.936166:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695874.936168:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6a00 x1874531139536768/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.936172:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695874.936174:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936175:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695874.936176:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936178:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695874.936180:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695874.936182:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695874.936185:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695874.936187:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695874.936191:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6a00 x1874531139536768/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695874.936199:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139536768:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695874.936203:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695874.936208:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695871/real 1787695874] req@ffff8b14466f7b80 x1874531139536896/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695874.936228:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695874.936230:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695874.936232:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547022c0. 00000400:00000200:1.0:1787695874.936234:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695874.936236:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695874.936238:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695874.936242:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7b80 x1874531139536896/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695874.936251:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7b80 x1874531139536896/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.936259:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695874.936261:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936263:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695874.936264:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695874.936268:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7b80 x1874531139536896/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.936275:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695874.936280:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7b80 x1874531139536896/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.936286:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695874.936289:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936291:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695874.936292:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936294:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695874.936296:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695874.936298:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695874.936302:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695874.936305:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695874.936316:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7b80 x1874531139536896/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695874.936323:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139536896:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695874.936326:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695874.936327:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695874.936330:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5880 x1874531139537024/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.936334:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695874.936336:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695874.936337:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695874.936340:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695874.936342:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695874.936343:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695874.936345:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695874.936347:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675400. 02000000:00000001:1.0:1787695874.936350:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695874.936352:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695874.936353:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5880. 00000100:00000001:1.0:1787695874.936356:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695874.936357:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936359:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695874.936382:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6680 x1874531139537152/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.936391:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695874.936394:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695874.936397:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545000. 02000000:00000001:1.0:1787695874.936402:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695874.936404:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695874.936405:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695874.936407:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695874.936409:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:1.0:1787695874.936413:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695874.936418:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695874.936420:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6680. 00000100:00000001:1.0:1787695874.936422:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695874.936422:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936424:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695874.936428:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5c00 x1874531139537280/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.936435:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695874.936436:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695874.936438:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544c00. 02000000:00000001:1.0:1787695874.936442:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695874.936443:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695874.936445:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695874.936447:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695874.936449:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676000. 02000000:00000001:1.0:1787695874.936453:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695874.936455:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695874.936456:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5c00. 00000100:00000001:1.0:1787695874.936458:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695874.936460:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936462:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695874.936466:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6a00 x1874531139536768/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.936473:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695874.936475:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695874.936476:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695874.936479:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695874.936480:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695874.936481:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695874.936482:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695874.936483:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:1.0:1787695874.936485:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695874.936486:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695874.936487:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6a00. 00000100:00000001:1.0:1787695874.936488:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695874.936489:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936495:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695874.936497:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7b80 x1874531139536896/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.936502:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695874.936503:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695874.936504:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695874.936506:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695874.936507:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695874.936508:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695874.936509:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695874.936510:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:1.0:1787695874.936511:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695874.936512:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695874.936513:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7b80. 00000100:00000001:1.0:1787695874.936514:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695874.936515:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936516:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695874.936517:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695874.936518:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695874.936519:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695874.936522:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695871/real 1787695874] req@ffff8b14466f7800 x1874531139537408/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695874.936528:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695874.936529:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695874.936531:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695874.936532:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695874.936533:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695874.936535:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695874.936537:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7800 x1874531139537408/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695874.936543:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7800 x1874531139537408/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.936547:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695874.936551:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936552:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695874.936553:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695874.936556:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7800 x1874531139537408/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.936561:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695874.936563:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7800 x1874531139537408/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.936567:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695874.936569:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936570:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695874.936570:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936572:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695874.936573:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695874.936574:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695874.936576:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695874.936578:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695874.936593:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7800 x1874531139537408/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695874.936599:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139537408:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695874.936601:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936602:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695874.936604:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7800 x1874531139537408/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695896 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695874.936608:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695874.936609:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695874.936610:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695874.936613:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695874.936614:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695874.936618:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695874.936620:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695874.936621:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:1.0:1787695874.936623:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695874.936624:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695874.936625:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7800. 00000100:00000001:1.0:1787695874.936626:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695874.936627:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936629:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695874.936636:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695874.936637:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695874.936639:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695874.936640:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695874.998478:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695874.998485:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695874.998487:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695874.998491:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695874.998494:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695874.998497:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695874.998499:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695874.998501:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695874.998503:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695875.638463:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695875.638469:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695875.638473:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695875.638474:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695875.638476:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695875.638477:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695875.638479:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695875.638480:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695875.638483:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695875.638484:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695875.638486:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695875.638487:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695875.638539:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695875.638542:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695875.638545:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695876.022468:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695876.022475:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695876.022477:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695876.022480:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695876.022483:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695876.022486:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695876.022488:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695876.022490:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695876.022491:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695876.662470:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695876.662476:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695876.662481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695876.662482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695876.662484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695876.662487:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695876.662489:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695876.662494:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695876.662496:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695876.662498:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695876.662500:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695876.662502:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695876.662503:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695876.662505:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695876.662527:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695876.662529:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695876.662530:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695876.662531:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695876.662532:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695876.662533:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695876.662534:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695876.662535:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695876.662536:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695876.662538:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695876.662540:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695876.662540:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695876.662542:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695876.662544:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695876.662544:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695876.662545:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695876.662546:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695876.662547:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695876.662548:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695876.662572:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695876.662574:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695876.662575:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695876.662639:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695876.662642:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695876.662645:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787695876.982476:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787695876.982487:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787695876.982490:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787695876.982494:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695876.982500:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787695876.982506:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3764 00000400:00000200:0.0:1787695876.982513:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695876.982520:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:0.0:1787695876.982535:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1787695876.982540:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.982558:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695876.982562:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.982565:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:0.0:1787695876.982570:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1787695876.982572:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:0.0:1787695876.982575:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1787695876.982579:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:0.0:1787695876.982582:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:0.0:1787695876.982586:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:0.0:1787695876.982590:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.982592:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695876.982594:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.982597:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:0.0:1787695876.982601:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:0.0:1787695876.982605:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695876.982609:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695876.982612:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787695876.982617:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695876.982620:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695876.982623:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787695876.982626:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1787695876.982633:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b145113d500. 00000020:00000040:0.0:1787695876.982637:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:0.0:1787695876.982651:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787695876.982653:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695876.982657:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695876.982659:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787695876.982663:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695876.982670:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f7400. 00000100:00000001:0.0:1787695876.982676:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695876.982689:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b145113d500 x1874531139537536/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695876.982697:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695876.982703:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695876.982708:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000dcd635e5] to pc [ptlrpcd_rcv+-1] req@ffff8b145113d500 x1874531139537536/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695876.982734:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695876.982737:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787695876.982741:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787695876.982742:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787695876.982744:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695876.982747:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787695876.982750:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3764 00000400:00000200:0.0:1787695876.982755:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695876.982759:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:0.0:1787695876.982762:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1787695876.982766:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.982768:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695876.982771:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1787695876.982772:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.982775:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695876.982776:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1787695876.982778:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695876.982780:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:0.0:1787695876.982781:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:0.0:1787695876.982784:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1787695876.982786:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:0.0:1787695876.982789:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:1.0:1787695876.982789:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b145113d500 x1874531139537536/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1787695876.982791:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:0.0:1787695876.982798:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.982800:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695876.982805:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.982813:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:0.0:1787695876.982815:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:0.0:1787695876.982819:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695876.982822:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695876.982824:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787695876.982829:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695876.982831:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695876.982833:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787695876.982837:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787695876.982841:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695876.982843:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1787695876.982846:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b145113ea00. 00000020:00000040:0.0:1787695876.982849:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:0.0:1787695876.982852:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787695876.982853:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695876.982855:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695876.982856:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787695876.982858:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695876.982862:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f5000. 00000100:00000001:0.0:1787695876.982866:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695876.982872:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b145113ea00 x1874531139537664/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695876.982887:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695876.982889:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695876.982891:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695876.982895:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695876.982898:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1787695876.982902:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000003e411b7] to pc [ptlrpcd_rcv+-1] req@ffff8b145113ea00 x1874531139537664/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695876.982914:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787695876.982915:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695876.982922:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1787695876.982923:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695876.982930:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787695876.982932:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b145113d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139537536:192.168.201.140@tcp:250:kworker.0 00000100:00000001:0.0:1787695876.982933:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787695876.982937:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695876.982940:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695876.982945:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695876.982949:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695876.982952:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787695876.982963:0:38:0:(import.c:485:import_select_connection()) Process entered 02000000:00000010:1.0:1787695876.982966:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547c00. 00000100:00080000:0.0:1787695876.982969:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3764 02000000:00000001:1.0:1787695876.982971:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695876.982977:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695876.982982:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695876.982984:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000010:1.0:1787695876.982988:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000100:00080000:0.0:1787695876.982990:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1787695876.982996:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000010:1.0:1787695876.982999:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000001:0.0:1787695876.983000:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695876.983005:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.983009:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000200:1.0:1787695876.983010:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139537536, portal 25 00000020:00000001:0.0:1787695876.983013:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1787695876.983016:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:0.0:1787695876.983018:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695876.983021:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000040:0.0:1787695876.983022:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000100:00000001:1.0:1787695876.983026:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615518108701952 : -128555600849664 : ffff8b145113d500) 00000020:00000001:0.0:1787695876.983028:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:0.0:1787695876.983033:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:1.0:1787695876.983038:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b145113d500 x1874531139537536/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695901 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695876.983041:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.983047:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695876.983052:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.983058:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695876.983058:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:0.0:1787695876.983064:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:0.0:1787695876.983069:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695876.983072:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695876.983074:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787695876.983078:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695876.983080:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695876.983081:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787695876.983084:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787695876.983086:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695876.983088:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1787695876.983092:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b145113d180. 00000020:00000040:0.0:1787695876.983101:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:0.0:1787695876.983103:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787695876.983105:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695876.983107:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695876.983109:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787695876.983111:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695876.983116:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f5800. 00000100:00000001:0.0:1787695876.983120:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695876.983126:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b145113d180 x1874531139537792/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695876.983144:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695876.983146:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695876.983150:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ed1e9b5e] to pc [ptlrpcd_rcv+-1] req@ffff8b145113d180 x1874531139537792/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695876.983158:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695876.983162:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787695876.983165:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787695876.983167:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787695876.983169:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695876.983171:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787695876.983174:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3764 00000400:00000200:0.0:1787695876.983191:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695876.983195:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:0.0:1787695876.983198:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1787695876.983201:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.983203:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695876.983205:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.983207:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:0.0:1787695876.983209:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1787695876.983211:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:0.0:1787695876.983232:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1787695876.983234:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:0.0:1787695876.983236:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:0.0:1787695876.983238:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:0.0:1787695876.983241:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.983243:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695876.983245:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.983248:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:0.0:1787695876.983250:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:0.0:1787695876.983253:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695876.983256:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695876.983258:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787695876.983261:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695876.983263:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695876.983265:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787695876.983267:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787695876.983270:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695876.983271:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1787695876.983275:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b145113e300. 00000020:00000040:0.0:1787695876.983277:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:0.0:1787695876.983280:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787695876.983281:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695876.983283:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695876.983285:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787695876.983286:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695876.983290:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f5c00. 00000100:00000001:0.0:1787695876.983293:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695876.983298:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b145113e300 x1874531139537920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695876.983305:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695876.983307:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695876.983311:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a6ea814f] to pc [ptlrpcd_rcv+-1] req@ffff8b145113e300 x1874531139537920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695876.983325:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:0.0:1787695876.983327:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695876.983332:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00080000:0.0:1787695876.983333:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787695876.983339:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000200:1.0:1787695876.983341:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139537536, offset 0 00000100:00000001:0.0:1787695876.983342:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787695876.983345:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695876.983349:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1787695876.983354:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00080000:0.0:1787695876.983355:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3764 00000400:00000200:0.0:1787695876.983403:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695876.983409:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:0.0:1787695876.983413:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1787695876.983414:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:0.0:1787695876.983420:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.983426:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695876.983426:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000040:0.0:1787695876.983433:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.983437:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695876.983437:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:0.0:1787695876.983442:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:1.0:1787695876.983445:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000020:00000040:0.0:1787695876.983446:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:0.0:1787695876.983450:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:1.0:1787695876.983453:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000020:00000040:0.0:1787695876.983454:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000400:00000200:1.0:1787695876.983459:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000020:00000001:0.0:1787695876.983460:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:0.0:1787695876.983470:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:0.0:1787695876.983475:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.983480:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695876.983481:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17009:17009:256:4294967295] 192.168.201.140@tcp LPNI seq info [17008:17008:8:4294967295] 00000100:00000040:0.0:1787695876.983486:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.983491:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695876.983492:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000040:0.0:1787695876.983499:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:0.0:1787695876.983503:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695876.983508:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695876.983512:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 02000000:00000001:0.0:1787695876.983519:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:1.0:1787695876.983523:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 02000000:00000001:0.0:1787695876.983525:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695876.983530:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695876.983532:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000010:1.0:1787695876.983532:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000008:00100000:0.0:1787695876.983537:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787695876.983542:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695876.983556:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:0.0:1787695876.983558:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695876.983564:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000100:00000010:0.0:1787695876.983567:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b145113f800. 00000020:00000040:0.0:1787695876.983573:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:0.0:1787695876.983579:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000200:1.1:1787695876.983579:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:0.0:1787695876.983582:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695876.983586:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.1:1787695876.983586:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 02000000:00000001:0.0:1787695876.983590:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787695876.983594:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787695876.983596:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 02000000:00000010:0.0:1787695876.983600:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f5400. 00000800:00000010:1.0:1787695876.983601:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000100:00000001:0.0:1787695876.983607:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695876.983611:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00100000:0.0:1787695876.983616:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b145113f800 x1874531139538048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695876.983630:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1787695876.983630:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00000001:0.0:1787695876.983635:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695876.983641:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000100:00000040:0.0:1787695876.983643:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005cf982f0] to pc [ptlrpcd_rcv+-1] req@ffff8b145113f800 x1874531139538048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1787695876.983646:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695876.983653:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787695876.983659:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695876.983663:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b145113d500 x1874531139537536/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695901 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787695876.983672:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787695876.983680:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787695876.983682:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787695876.983682:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1787695876.983687:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1787695876.983690:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b145113d500 x1874531139537536/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695901 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695876.983692:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787695876.983699:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3764 00000100:00000001:1.0:1787695876.983705:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695876.983710:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695876.983711:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1787695876.983718:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695876.983721:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695876.983722:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1787695876.983729:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695876.983735:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298448697 00000100:00000040:0.0:1787695876.983737:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.983741:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695876.983744:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298443576 00000100:00000040:0.0:1787695876.983745:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.983751:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695876.983753:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298448697 00000020:00000001:0.0:1787695876.983754:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1787695876.983759:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000100:00000001:1.0:1787695876.983760:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298443576 00000020:00000001:0.0:1787695876.983762:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695876.983766:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:0.0:1787695876.983767:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000020:00000001:0.0:1787695876.983773:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:1.0:1787695876.983773:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695876.983779:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:1.0:1787695876.983780:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:0.0:1787695876.983783:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.983786:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1787695876.983790:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695876/real 1787695876] req@ffff8b145113d500 x1874531139537536/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695876.983791:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695876.983794:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:0.0:1787695876.983799:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:0.0:1787695876.983806:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695876.983810:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695876.983812:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787695876.983818:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:1.0:1787695876.983820:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1787695876.983822:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787695876.983824:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787695876.983827:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:1.0:1787695876.983829:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000008:00100000:0.0:1787695876.983832:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1787695876.983834:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000008:00000001:0.0:1787695876.983837:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695876.983839:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:0.0:1787695876.983841:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695876.983844:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000010:0.0:1787695876.983848:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b145113dc00. 00000100:00000200:1.0:1787695876.983852:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b145113d500 x1874531139537536/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1787695876.983854:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:0.0:1787695876.983860:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787695876.983863:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695876.983868:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695876.983873:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:1.0:1787695876.983874:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b145113d500 x1874531139537536/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787695876.983877:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695876.983882:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f4400. 00000100:00000001:1.0:1787695876.983890:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695876.983894:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695876.983896:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695876.983897:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695876.983903:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695876.983908:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b145113dc00 x1874531139538176/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787695876.983912:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b145113d500 x1874531139537536/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.983923:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695876.983932:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1787695876.983933:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b145113d500 x1874531139537536/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695876.983937:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695876.983948:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000002b99ca3] to pc [ptlrpcd_rcv+-1] req@ffff8b145113dc00 x1874531139538176/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.983949:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695876.983955:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.983958:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695876.983961:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.983966:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787695876.983970:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695876.983971:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695876.983977:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695876.983977:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695876.983982:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00080000:0.0:1787695876.983986:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787695876.983987:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1787695876.983991:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:1.0:1787695876.983995:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b145113d500 x1874531139537536/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787695876.983996:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695876.984002:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787695876.984009:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3764 00000100:00100000:1.0:1787695876.984010:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b145113d500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139537536:192.168.201.140@tcp:250:kworker.0 00000100:00000001:1.0:1787695876.984018:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1787695876.984022:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695876.984023:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695876.984028:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b145113d500 x1874531139537536/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695876.984032:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695876.984037:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00080000:0.0:1787695876.984039:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1787695876.984040:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:0.0:1787695876.984044:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000010:1.0:1787695876.984049:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547c00. 00000100:00000001:0.0:1787695876.984050:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695876.984055:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695876.984056:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695876.984059:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:0.0:1787695876.984061:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695876.984062:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:1.0:1787695876.984066:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000020:00000001:0.0:1787695876.984068:0:38:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000010:1.0:1787695876.984069:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f7400. 00000020:00000040:0.0:1787695876.984071:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 02000000:00000001:1.0:1787695876.984074:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:0.0:1787695876.984076:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1787695876.984077:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695876.984081:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b145113d500. 00000020:00000040:0.0:1787695876.984082:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000100:00000001:1.0:1787695876.984083:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695876.984086:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787695876.984088:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:1.0:1787695876.984089:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787695876.984095:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:1.0:1787695876.984097:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695876.984101:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695876.984102:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695876.984105:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695876.984108:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695876.984109:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695876.984111:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695876.984114:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b145113ea00 x1874531139537664/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695876.984123:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695876.984123:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695876.984125:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695876.984130:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:1.0:1787695876.984131:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695876.984133:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:0.0:1787695876.984137:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 02000000:00000001:1.0:1787695876.984137:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695876.984143:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b145113ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139537664:192.168.201.140@tcp:38:kworker.0 00000100:00080000:0.0:1787695876.984146:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695876.984147:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695876.984151:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787695876.984153:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695876.984153:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695876.984160:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695876.984160:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695876.984165:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547c00. 02000000:00000001:0.0:1787695876.984166:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695876.984167:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695876.984173:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695876.984174:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695876.984177:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:1.0:1787695876.984179:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000008:00100000:0.0:1787695876.984183:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000010:1.0:1787695876.984184:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000008:00000001:0.0:1787695876.984190:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695876.984191:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139537664, portal 10 00000100:00000001:0.0:1787695876.984193:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695876.984199:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000010:0.0:1787695876.984201:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b145113c000. 00000100:00000001:1.0:1787695876.984201:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615518108707328 : -128555600844288 : ffff8b145113ea00) 00000020:00000040:0.0:1787695876.984207:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000040:1.0:1787695876.984212:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b145113ea00 x1874531139537664/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695901 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787695876.984214:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787695876.984218:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695876.984224:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787695876.984224:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1787695876.984229:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1787695876.984229:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695876.984234:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 02000000:00000001:0.0:1787695876.984235:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695876.984237:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139537664, offset 0 00000400:00000200:1.0:1787695876.984243:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 02000000:00000010:0.0:1787695876.984244:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f7400. 00000100:00000001:0.0:1787695876.984251:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695876.984251:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695876.984259:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00100000:0.0:1787695876.984262:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b145113c000 x1874531139538304/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695876.984266:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695876.984273:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:0.0:1787695876.984277:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1787695876.984279:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000001:0.0:1787695876.984283:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695876.984285:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000040:0.0:1787695876.984292:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7e024f7] to pc [ptlrpcd_rcv+-1] req@ffff8b145113c000 x1874531139538304/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695876.984293:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17010:17010:256:4294967295] 192.168.201.140@tcp LPNI seq info [17009:17009:8:4294967295] 00000400:00000200:1.0:1787695876.984308:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695876.984316:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695876.984322:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000040:0.0:1787695876.984323:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:0.0:1787695876.984330:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.0:1787695876.984330:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000040:0.0:1787695876.984336:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:1.0:1787695876.984337:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695876.984342:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695876.984347:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000100:00000040:0.0:1787695876.984349:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1787695876.984354:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:0.0:1787695876.984357:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3774) 00000800:00000200:1.1:1787695876.984357:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1787695876.984389:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695876.984392:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695876.984396:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695876.984399:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695876.984402:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695876.984404:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695876.984406:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695876.984409:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b145113ea00 x1874531139537664/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695901 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.984424:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695876.984427:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b145113ea00 x1874531139537664/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695901 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.984431:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695876.984433:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695876.984436:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695876.984439:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000400:00000200:1.0:1787695876.984441:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695876.984443:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298438599 00000100:00000001:1.0:1787695876.984445:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298448697 00000100:00000001:1.0:1787695876.984447:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298438599 00000100:00000001:1.0:1787695876.984449:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298448697 00000100:00000001:1.0:1787695876.984450:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695876.984452:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695876.984453:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695876.984456:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695876/real 1787695876] req@ffff8b145113ea00 x1874531139537664/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695876.984461:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695876.984462:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695876.984463:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695876.984464:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695876.984465:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695876.984466:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695876.984468:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b145113ea00 x1874531139537664/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695876.984474:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b145113ea00 x1874531139537664/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.984478:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695876.984483:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.984485:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695876.984486:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695876.984488:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b145113ea00 x1874531139537664/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.984493:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695876.984495:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b145113ea00 x1874531139537664/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.984500:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695876.984502:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.984503:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695876.984504:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.984505:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695876.984506:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695876.984508:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695876.984510:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695876.984512:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695876.984516:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b145113ea00 x1874531139537664/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695876.984524:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b145113ea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139537664:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695876.984529:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695876.984531:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b145113d180 x1874531139537792/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.984538:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695876.984540:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695876.984542:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695876.984556:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695876.984559:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695876.984569:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b145113d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139537792:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695876.984572:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695876.984575:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695876.984577:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695876.984578:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695876.984582:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695876.984585:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695876.984589:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695876.984591:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695876.984594:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695876.984597:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139537792, portal 10 00000100:00000001:1.0:1787695876.984598:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695876.984600:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615518108701056 : -128555600850560 : ffff8b145113d180) 00000100:00000040:1.0:1787695876.984605:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b145113d180 x1874531139537792/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695901 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.984609:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695876.984611:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695876.984612:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695876.984614:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139537792, offset 0 00000400:00000200:1.0:1787695876.984617:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695876.984619:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695876.984622:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695876.984624:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695876.984626:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695876.984628:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695876.984630:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695876.984632:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17011:17011:256:4294967295] 192.168.201.140@tcp LPNI seq info [17010:17010:8:4294967295] 00000400:00000200:1.0:1787695876.984635:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695876.984640:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695876.984644:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695876.984647:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695876.984657:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695876.984660:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1787695876.984664:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695876.984667:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1787695876.984671:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695876.984673:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695876.984676:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695876.984680:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695876.984684:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695876.984686:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695876.984689:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695876.984693:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b145113d180 x1874531139537792/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695901 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.984700:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695876.984703:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b145113d180 x1874531139537792/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695901 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.984707:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695876.984708:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695876.984711:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695876.984714:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787695876.984716:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695876.984718:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298437447 00000100:00000001:1.0:1787695876.984720:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298448697 00000100:00000001:1.0:1787695876.984721:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298437447 00000100:00000001:1.0:1787695876.984722:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298448697 00000100:00000001:1.0:1787695876.984724:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695876.984726:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695876.984727:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695876.984730:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695876/real 1787695876] req@ffff8b145113d180 x1874531139537792/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695876.984734:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695876.984740:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695876.984741:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695876.984742:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695876.984744:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695876.984745:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695876.984748:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b145113d180 x1874531139537792/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695876.984753:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b145113d180 x1874531139537792/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.984757:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695876.984758:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.984759:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695876.984760:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695876.984762:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b145113d180 x1874531139537792/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.984766:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695876.984769:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b145113d180 x1874531139537792/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.984773:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695876.984775:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.984776:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695876.984777:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.984778:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695876.984779:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695876.984781:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695876.984783:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695876.984784:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695876.984786:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b145113d180 x1874531139537792/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695876.984798:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b145113d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139537792:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695876.984801:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695876.984803:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b145113e300 x1874531139537920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.984807:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695876.984808:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695876.984809:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695876.984811:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695876.984812:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695876.984814:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b145113e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139537920:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695876.984816:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695876.984817:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695876.984819:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695876.984820:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695876.984822:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695876.984823:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695876.984826:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695876.984828:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695876.984830:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695876.984832:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139537920, portal 4 00000100:00000001:1.0:1787695876.984834:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695876.984835:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615518108705536 : -128555600846080 : ffff8b145113e300) 00000100:00000040:1.0:1787695876.984839:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b145113e300 x1874531139537920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695901 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.984844:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695876.984845:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695876.984846:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695876.984848:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139537920, offset 0 00000400:00000200:1.0:1787695876.984851:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695876.984856:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695876.984859:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695876.984861:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695876.984863:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695876.984865:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695876.984866:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695876.984868:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17012:17012:256:4294967295] 192.168.201.140@tcp LPNI seq info [17011:17011:8:4294967295] 00000400:00000200:1.0:1787695876.984871:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695876.984876:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695876.984878:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695876.984880:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695876.984883:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695876.984885:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1787695876.984888:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695876.984890:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1787695876.984892:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695876.984894:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695876.984895:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695876.984898:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695876.984900:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695876.984901:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695876.984904:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695876.984908:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b145113e300 x1874531139537920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695901 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.984916:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695876.984919:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b145113e300 x1874531139537920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695901 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.984926:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695876.984927:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695876.984931:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695876.984934:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695876.984937:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695876.984946:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298437477 00000100:00000001:1.0:1787695876.984949:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298448697 00000100:00000001:1.0:1787695876.984952:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298437477 00000100:00000001:1.0:1787695876.984953:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298448697 00000100:00000001:1.0:1787695876.984955:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695876.984956:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695876.984958:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695876.984961:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695876/real 1787695876] req@ffff8b145113e300 x1874531139537920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695876.984967:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695876.984968:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695876.984969:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695876.984971:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695876.984972:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695876.984974:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695876.984976:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b145113e300 x1874531139537920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695876.984982:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b145113e300 x1874531139537920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.984986:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695876.984987:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.984988:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695876.984989:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695876.984991:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b145113e300 x1874531139537920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.984995:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695876.984998:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b145113e300 x1874531139537920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985002:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695876.985007:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.985009:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695876.985010:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.985011:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695876.985012:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695876.985014:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695876.985016:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695876.985017:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695876.985020:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b145113e300 x1874531139537920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695876.985025:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b145113e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139537920:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695876.985027:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695876.985029:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b145113f800 x1874531139538048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985034:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695876.985035:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695876.985036:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695876.985037:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695876.985038:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695876.985040:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b145113f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139538048:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695876.985042:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695876.985043:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695876.985044:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695876.985045:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695876.985047:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544c00. 02000000:00000001:1.0:1787695876.985049:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695876.985051:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695876.985052:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695876.985055:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695876.985057:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139538048, portal 4 00000100:00000001:1.0:1787695876.985061:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695876.985062:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615518108710912 : -128555600840704 : ffff8b145113f800) 00000100:00000040:1.0:1787695876.985067:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b145113f800 x1874531139538048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695901 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985071:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695876.985072:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695876.985074:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695876.985076:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139538048, offset 0 00000400:00000200:1.0:1787695876.985078:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695876.985081:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695876.985083:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695876.985085:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695876.985087:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695876.985089:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695876.985090:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695876.985092:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17013:17013:256:4294967295] 192.168.201.140@tcp LPNI seq info [17012:17012:8:4294967295] 00000400:00000200:1.0:1787695876.985095:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695876.985100:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695876.985103:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695876.985105:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695876.985107:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695876.985109:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1787695876.985112:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695876.985114:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1787695876.985117:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695876.985118:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695876.985120:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695876.985122:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695876.985124:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695876.985125:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695876.985127:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695876.985129:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b145113f800 x1874531139538048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695901 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985141:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695876.985144:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b145113f800 x1874531139538048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695901 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985148:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695876.985149:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695876.985152:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695876.985154:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695876.985156:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695876.985158:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298437149 00000100:00000001:1.0:1787695876.985160:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298448697 00000100:00000001:1.0:1787695876.985161:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298437149 00000100:00000001:1.0:1787695876.985163:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298448697 00000100:00000001:1.0:1787695876.985164:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695876.985165:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695876.985167:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695876.985169:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695876/real 1787695876] req@ffff8b145113f800 x1874531139538048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695876.985173:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695876.985174:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695876.985175:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695876.985176:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695876.985187:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695876.985192:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695876.985194:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b145113f800 x1874531139538048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695876.985199:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b145113f800 x1874531139538048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985203:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695876.985208:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.985209:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695876.985210:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695876.985213:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b145113f800 x1874531139538048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985218:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695876.985220:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b145113f800 x1874531139538048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985224:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695876.985226:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.985227:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695876.985228:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.985229:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695876.985230:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695876.985231:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695876.985233:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695876.985234:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695876.985236:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b145113f800 x1874531139538048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695876.985241:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b145113f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139538048:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695876.985244:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695876.985247:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b145113dc00 x1874531139538176/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985254:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695876.985255:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695876.985257:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695876.985259:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695876.985261:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695876.985269:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b145113dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139538176:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695876.985272:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695876.985274:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695876.985276:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695876.985278:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695876.985281:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545000. 02000000:00000001:1.0:1787695876.985283:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695876.985287:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695876.985289:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695876.985293:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695876.985297:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139538176, portal 10 00000100:00000001:1.0:1787695876.985298:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695876.985300:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615518108703744 : -128555600847872 : ffff8b145113dc00) 00000100:00000040:1.0:1787695876.985305:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b145113dc00 x1874531139538176/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695901 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985309:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695876.985311:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695876.985312:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695876.985314:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139538176, offset 0 00000400:00000200:1.0:1787695876.985317:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695876.985320:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695876.985322:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695876.985324:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695876.985326:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695876.985327:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695876.985329:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695876.985332:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17014:17014:256:4294967295] 192.168.201.140@tcp LPNI seq info [17013:17013:8:4294967295] 00000400:00000200:1.0:1787695876.985337:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695876.985343:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695876.985348:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695876.985352:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695876.985362:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695876.985395:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1787695876.985399:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695876.985401:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1787695876.985405:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695876.985408:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695876.985411:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695876.985416:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695876.985420:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695876.985423:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695876.985427:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695876.985431:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b145113dc00 x1874531139538176/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695901 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985437:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695876.985439:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b145113dc00 x1874531139538176/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695901 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985444:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695876.985445:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695876.985448:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695876.985450:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695876.985452:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695876.985454:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298437336 00000100:00000001:1.0:1787695876.985456:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298448697 00000100:00000001:1.0:1787695876.985457:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298437336 00000100:00000001:1.0:1787695876.985459:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298448697 00000100:00000001:1.0:1787695876.985460:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695876.985461:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695876.985463:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695876.985465:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695876/real 1787695876] req@ffff8b145113dc00 x1874531139538176/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695876.985470:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695876.985476:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695876.985477:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695876.985479:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695876.985480:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695876.985481:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695876.985484:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b145113dc00 x1874531139538176/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695876.985489:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b145113dc00 x1874531139538176/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985493:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695876.985494:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.985495:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695876.985496:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695876.985498:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b145113dc00 x1874531139538176/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985502:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695876.985505:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b145113dc00 x1874531139538176/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985509:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695876.985512:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.985513:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695876.985514:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.985516:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695876.985518:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695876.985521:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695876.985524:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695876.985526:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695876.985530:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b145113dc00 x1874531139538176/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695876.985563:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b145113dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139538176:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695876.985567:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.985569:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695876.985573:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b145113ea00 x1874531139537664/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985580:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695876.985582:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695876.985584:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695876.985588:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695876.985590:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695876.985591:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695876.985594:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695876.985596:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f5000. 02000000:00000001:1.0:1787695876.985599:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695876.985601:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695876.985602:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b145113ea00. 00000100:00000001:1.0:1787695876.985605:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695876.985607:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.985609:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695876.985612:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b145113d180 x1874531139537792/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985619:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695876.985620:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695876.985622:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695876.985625:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695876.985627:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695876.985628:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695876.985630:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695876.985632:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f5800. 02000000:00000001:1.0:1787695876.985634:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695876.985636:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695876.985637:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b145113d180. 00000100:00000001:1.0:1787695876.985646:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695876.985647:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.985649:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695876.985653:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b145113e300 x1874531139537920/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985660:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695876.985661:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695876.985663:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695876.985666:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695876.985667:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695876.985669:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695876.985671:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695876.985672:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f5c00. 02000000:00000001:1.0:1787695876.985675:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695876.985676:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695876.985677:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b145113e300. 00000100:00000001:1.0:1787695876.985680:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695876.985681:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.985683:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695876.985687:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b145113f800 x1874531139538048/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985692:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695876.985693:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695876.985694:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544c00. 02000000:00000001:1.0:1787695876.985695:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695876.985696:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695876.985697:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695876.985698:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695876.985699:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f5400. 02000000:00000001:1.0:1787695876.985700:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695876.985701:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695876.985702:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b145113f800. 00000100:00000001:1.0:1787695876.985703:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695876.985704:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.985705:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695876.985707:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b145113dc00 x1874531139538176/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985716:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695876.985717:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695876.985718:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545000. 02000000:00000001:1.0:1787695876.985721:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695876.985722:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695876.985722:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695876.985724:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695876.985725:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f4400. 02000000:00000001:1.0:1787695876.985726:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695876.985727:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695876.985728:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b145113dc00. 00000100:00000001:1.0:1787695876.985729:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695876.985730:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.985731:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.985734:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695876.985735:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695876.985736:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695876.985737:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695876.985738:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695876.985741:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b145113c000 x1874531139538304/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985745:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695876.985746:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695876.985748:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695876.985749:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695876.985750:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695876.985752:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b145113c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139538304:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695876.985755:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695876.985756:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695876.985757:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695876.985758:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695876.985761:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545000. 02000000:00000001:1.0:1787695876.985762:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695876.985767:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695876.985769:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695876.985771:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695876.985774:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139538304, portal 10 00000100:00000001:1.0:1787695876.985775:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695876.985776:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615518108696576 : -128555600855040 : ffff8b145113c000) 00000100:00000040:1.0:1787695876.985780:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b145113c000 x1874531139538304/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695901 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985785:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695876.985786:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695876.985788:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695876.985789:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139538304, offset 0 00000400:00000200:1.0:1787695876.985792:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695876.985796:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695876.985799:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695876.985802:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695876.985805:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695876.985808:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695876.985810:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695876.985813:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17015:17015:256:4294967295] 192.168.201.140@tcp LPNI seq info [17014:17014:8:4294967295] 00000400:00000200:1.0:1787695876.985817:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695876.985824:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695876.985829:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695876.985833:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695876.985837:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695876.985840:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000200:1.1:1787695876.985845:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695876.985848:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000100:1.0:1787695876.985852:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695876.985853:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695876.985855:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695876.985857:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695876.985860:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695876.985865:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695876.985867:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695876.985870:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b145113c000 x1874531139538304/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695901 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985876:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695876.985878:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b145113c000 x1874531139538304/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695901 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985882:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695876.985883:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695876.985886:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695876.985888:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695876.985891:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695876.985892:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298437314 00000100:00000001:1.0:1787695876.985894:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298448697 00000100:00000001:1.0:1787695876.985895:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298437314 00000100:00000001:1.0:1787695876.985897:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298448697 00000100:00000001:1.0:1787695876.985898:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695876.985899:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695876.985901:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695876.985905:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695876/real 1787695876] req@ffff8b145113c000 x1874531139538304/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695876.985913:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695876.985914:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695876.985916:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695876.985919:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695876.985921:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695876.985923:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695876.985926:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b145113c000 x1874531139538304/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695876.985935:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b145113c000 x1874531139538304/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985947:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695876.985948:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.985950:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695876.985952:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695876.985956:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b145113c000 x1874531139538304/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985963:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695876.985966:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b145113c000 x1874531139538304/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.985972:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695876.985975:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.985976:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695876.985977:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.985979:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695876.985981:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695876.985983:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695876.985986:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695876.985988:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695876.985993:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b145113c000 x1874531139538304/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695876.986001:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b145113c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139538304:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695876.986004:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.986007:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695876.986010:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b145113c000 x1874531139538304/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695901 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695876.986017:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695876.986022:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695876.986024:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545000. 02000000:00000001:1.0:1787695876.986026:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695876.986027:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695876.986028:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695876.986029:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695876.986030:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f7400. 02000000:00000001:1.0:1787695876.986032:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695876.986033:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695876.986034:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b145113c000. 00000100:00000001:1.0:1787695876.986036:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695876.986037:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.986038:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695876.986041:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695876.986042:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695876.986043:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695876.986044:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695877.046485:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695877.046494:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695877.046496:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695877.046503:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695877.046506:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695877.046533:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695877.046535:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695877.046538:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695877.046540:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695877.686496:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695877.686506:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695877.686514:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695877.686516:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695877.686521:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695877.686523:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695877.686528:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695877.686530:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695877.686550:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695877.686551:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695877.686556:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695877.686558:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695877.686642:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695877.686646:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695877.686650:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695878.070467:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695878.070474:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695878.070475:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695878.070480:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695878.070482:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695878.070485:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695878.070487:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695878.070489:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695878.070491:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695878.710463:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695878.710468:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695878.710472:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695878.710473:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695878.710476:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695878.710477:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695878.710479:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695878.710481:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695878.710483:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695878.710498:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695878.710501:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695878.710502:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695878.710557:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695878.710562:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695878.710564:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695879.094481:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695879.094487:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695879.094489:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695879.094494:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695879.094496:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695879.094499:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695879.094502:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695879.094504:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695879.094505:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695879.734442:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695879.734448:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695879.734452:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695879.734453:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695879.734456:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695879.734457:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695879.734460:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695879.734461:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695879.734463:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695879.734464:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695879.734466:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695879.734467:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695879.734537:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695879.734542:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695879.734546:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695880.118514:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695880.118521:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695880.118524:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695880.118530:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695880.118549:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695880.118553:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695880.118556:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695880.118558:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695880.118561:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695880.758500:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695880.758506:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695880.758512:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695880.758513:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695880.758515:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695880.758516:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695880.758519:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695880.758520:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695880.758522:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695880.758523:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695880.758526:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695880.758526:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695880.758583:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695880.758586:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695880.758589:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695881.142500:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695881.142512:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695881.142515:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695881.142524:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695881.142549:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695881.142553:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695881.142556:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695881.142558:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695881.142561:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695881.782501:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695881.782509:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695881.782514:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695881.782515:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695881.782519:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695881.782520:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695881.782523:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695881.782524:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695881.782527:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695881.782531:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695881.782533:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695881.782558:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695881.782560:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695881.782562:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695881.782565:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695881.782568:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695881.782569:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695881.782571:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695881.782574:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695881.782575:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695881.782661:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695881.782667:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695881.782670:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787695882.102616:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787695882.102630:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787695882.102634:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787695882.102637:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695882.102643:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787695882.102650:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3769 00000400:00000200:0.0:1787695882.102659:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695882.102665:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:0.0:1787695882.102669:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1787695882.102672:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695882.102676:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695882.102678:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695882.102681:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:0.0:1787695882.102685:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1787695882.102688:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:0.0:1787695882.102690:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1787695882.102694:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:0.0:1787695882.102697:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:0.0:1787695882.102700:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:0.0:1787695882.102703:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695882.102720:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695882.102722:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695882.102725:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:0.0:1787695882.102728:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:0.0:1787695882.102731:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695882.102734:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695882.102737:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787695882.102741:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695882.102744:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695882.102747:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787695882.102750:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1787695882.102757:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b145113ce00. 00000020:00000040:0.0:1787695882.102761:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:0.0:1787695882.102764:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787695882.102766:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695882.102769:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695882.102772:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787695882.102778:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695882.102789:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f7400. 00000100:00000001:0.0:1787695882.102797:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695882.102809:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b145113ce00 x1874531139538432/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787695882.102821:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695882.102827:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695882.102831:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019501818] to pc [ptlrpcd_rcv+-1] req@ffff8b145113ce00 x1874531139538432/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695882.102856:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695882.102860:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787695882.102864:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787695882.102865:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787695882.102867:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695882.102878:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787695882.102887:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3769 00000400:00000200:0.0:1787695882.102893:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695882.102898:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:0.0:1787695882.102900:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1787695882.102905:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695882.102906:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787695882.102908:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695882.102913:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695882.102914:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787695882.102915:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695882.102919:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1787695882.102920:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000040:1.0:1787695882.102930:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b145113ce00 x1874531139538432/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1787695882.102932:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:0.0:1787695882.102936:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1787695882.102941:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:0.0:1787695882.102946:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000001:1.0:1787695882.102948:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1787695882.102951:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000001:1.0:1787695882.102952:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695882.102958:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695882.102959:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787695882.102963:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787695882.102965:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:0.0:1787695882.102969:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695882.102971:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695882.102974:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:0.0:1787695882.102978:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00100000:1.0:1787695882.102980:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b145113ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139538432:192.168.201.140@tcp:250:kworker.0 00000100:00080000:0.0:1787695882.102983:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695882.102987:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787695882.102988:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695882.102991:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695882.102992:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695882.102994:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695882.102996:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695882.102997:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787695882.103001:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695882.103002:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000010:1.0:1787695882.103004:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545000. 00000008:00100000:0.0:1787695882.103006:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787695882.103010:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787695882.103011:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695882.103016:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1787695882.103020:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b145113ed80. 00000020:00000040:0.0:1787695882.103024:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000400:00000200:1.0:1787695882.103024:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:0.0:1787695882.103029:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000010:1.0:1787695882.103032:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 02000000:00000001:0.0:1787695882.103037:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695882.103040:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695882.103041:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787695882.103043:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695882.103043:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 02000000:00000010:0.0:1787695882.103052:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f4400. 00000100:00000200:1.0:1787695882.103054:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139538432, portal 25 00000100:00000001:0.0:1787695882.103055:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695882.103061:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:0.0:1787695882.103063:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b145113ed80 x1874531139538560/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695882.103065:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615518108700160 : -128555600851456 : ffff8b145113ce00) 00000100:00000001:0.0:1787695882.103073:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1787695882.103077:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b145113ce00 x1874531139538432/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695882.103078:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695882.103084:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000005499392] to pc [ptlrpcd_rcv+-1] req@ffff8b145113ed80 x1874531139538560/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.103104:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787695882.103105:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695882.103107:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787695882.103110:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787695882.103111:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787695882.103113:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1787695882.103117:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695882.103120:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695882.103124:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139538432, offset 0 00000100:00000001:0.0:1787695882.103129:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1787695882.103130:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00080000:0.0:1787695882.103137:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3769 00000400:00000200:1.0:1787695882.103142:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695882.103160:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695882.103165:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695882.103169:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695882.103172:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695882.103174:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695882.103177:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17016:17016:256:4294967295] 192.168.201.140@tcp LPNI seq info [17015:17015:8:4294967295] 00000400:00000200:1.0:1787695882.103181:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695882.103187:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695882.103193:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000400:00000200:0.0:1787695882.103197:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:0.0:1787695882.103201:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1787695882.103202:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000040:0.0:1787695882.103210:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000800:00000010:1.0:1787695882.103210:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000100:00000001:0.0:1787695882.103216:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.0:1787695882.103220:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:0.0:1787695882.103222:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000200:1.0:1787695882.103225:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000100:00000001:0.0:1787695882.103226:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:0.0:1787695882.103231:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:0.0:1787695882.103234:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000800:00000200:1.1:1787695882.103235:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000020:00000001:0.0:1787695882.103237:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000200:1.1:1787695882.103239:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1787695882.103243:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695882.103245:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695882.103255:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695882.103259:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695882.103263:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695882.103265:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695882.103269:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695882.103272:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b145113ce00 x1874531139538432/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.103280:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695882.103283:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b145113ce00 x1874531139538432/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695907 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1787695882.103290:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000100:00000001:1.0:1787695882.103291:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695882.103294:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000001:1.0:1787695882.103295:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:0.0:1787695882.103305:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000001:1.0:1787695882.103307:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695882.103312:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695882.103318:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695882.103318:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:0.0:1787695882.103322:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695882.103323:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298453816 00000100:00000001:0.0:1787695882.103328:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:0.0:1787695882.103333:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00000001:1.0:1787695882.103335:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298448697 00000100:00080000:0.0:1787695882.103338:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695882.103342:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298453816 00000100:00000001:0.0:1787695882.103343:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695882.103349:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695882.103351:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298448697 02000000:00000001:0.0:1787695882.103353:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695882.103356:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695882.103358:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695882.103384:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695882.103390:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695882.103390:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000008:00100000:0.0:1787695882.103396:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787695882.103399:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695882/real 1787695882] req@ffff8b145113ce00 x1874531139538432/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1787695882.103403:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695882.103408:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1787695882.103415:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b145113fb80. 00000400:00000001:1.0:1787695882.103417:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000040:0.0:1787695882.103422:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000400:00000001:1.0:1787695882.103422:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695882.103425:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 02000000:00000001:0.0:1787695882.103427:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1787695882.103431:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 02000000:00000001:0.0:1787695882.103432:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695882.103437:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:1.0:1787695882.103437:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 02000000:00000001:0.0:1787695882.103441:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787695882.103443:0:2402:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:0.0:1787695882.103444:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1787695882.103450:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f5400. 00000100:00000200:1.0:1787695882.103451:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b145113ce00 x1874531139538432/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695882.103454:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787695882.103473:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b145113fb80 x1874531139538688/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1787695882.103482:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b145113ce00 x1874531139538432/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695882.103483:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695882.103486:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695882.103494:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000cd7e8e3e] to pc [ptlrpcd_rcv+-1] req@ffff8b145113fb80 x1874531139538688/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.103496:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695882.103499:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.103503:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695882.103506:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695882.103508:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695882.103512:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b145113ce00 x1874531139538432/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787695882.103520:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695882.103526:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787695882.103528:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787695882.103532:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:1.0:1787695882.103535:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b145113ce00 x1874531139538432/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787695882.103536:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695882.103543:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787695882.103549:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3769 00000400:00000200:0.0:1787695882.103554:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695882.103561:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:0.0:1787695882.103567:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:0.0:1787695882.103574:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695882.103574:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695882.103577:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695882.103579:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1787695882.103581:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695882.103583:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787695882.103585:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695882.103586:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787695882.103590:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695882.103591:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000020:00000040:0.0:1787695882.103594:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000100:00000001:1.0:1787695882.103595:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:0.0:1787695882.103599:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00080000:1.0:1787695882.103600:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000020:00000040:0.0:1787695882.103602:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000100:00080000:1.0:1787695882.103606:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000020:00000001:0.0:1787695882.103607:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:1.0:1787695882.103610:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:0.0:1787695882.103611:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:0.0:1787695882.103618:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695882.103619:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b145113ce00 x1874531139538432/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695882.103620:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695882.103624:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695882.103628:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:0.0:1787695882.103632:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:0.0:1787695882.103638:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695882.103642:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695882.103645:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787695882.103648:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1787695882.103649:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b145113ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139538432:192.168.201.140@tcp:250:kworker.0 02000000:00000001:0.0:1787695882.103650:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695882.103654:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695882.103658:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:0.0:1787695882.103661:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787695882.103663:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000008:00000001:0.0:1787695882.103665:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695882.103668:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695882.103670:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b145113ce00 x1874531139538432/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1787695882.103672:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b145113c000. 00000020:00000040:0.0:1787695882.103677:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000100:00000001:1.0:1787695882.103682:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787695882.103683:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695882.103685:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787695882.103686:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695882.103690:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000010:1.0:1787695882.103690:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545000. 02000000:00000001:0.0:1787695882.103692:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787695882.103695:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695882.103696:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695882.103699:0:2402:0:(genops.c:1217:class_import_put()) Process entered 02000000:00000010:0.0:1787695882.103700:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f5c00. 00000020:00000040:1.0:1787695882.103702:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000100:00000001:0.0:1787695882.103704:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787695882.103705:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695882.103709:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f7400. 00000100:00100000:0.0:1787695882.103713:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b145113c000 x1874531139538816/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787695882.103714:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695882.103717:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695882.103719:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b145113ce00. 00000100:00000001:1.0:1787695882.103724:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695882.103726:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695882.103728:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695882.103729:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695882.103731:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695882.103734:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e7e024f7] to pc [ptlrpcd_rcv+-1] req@ffff8b145113c000 x1874531139538816/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695882.103739:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695882.103741:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787695882.103744:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1787695882.103747:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787695882.103748:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787695882.103751:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787695882.103752:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3769 00000100:00000001:1.0:1787695882.103754:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:0.0:1787695882.103757:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695882.103758:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695882.103762:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1787695882.103765:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695882.103765:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695882.103768:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1787695882.103772:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695882.103774:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b145113ed80 x1874531139538560/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695882.103779:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695882.103783:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787695882.103786:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695882.103786:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695882.103791:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:0.0:1787695882.103794:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695882.103794:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695882.103796:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695882.103799:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b145113ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139538560:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695882.103801:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695882.103803:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695882.103806:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695882.103808:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:1.0:1787695882.103808:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695882.103811:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545000. 00000020:00000001:0.0:1787695882.103813:0:38:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1787695882.103814:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1787695882.103816:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:0.0:1787695882.103819:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1787695882.103822:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000400:00000200:1.0:1787695882.103823:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:0.0:1787695882.103826:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:0.0:1787695882.103829:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000400:00000010:1.0:1787695882.103829:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695882.103837:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000040:0.0:1787695882.103838:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695882.103844:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1787695882.103845:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139538560, portal 10 00000100:00000040:0.0:1787695882.103848:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695882.103850:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787695882.103854:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695882.103854:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615518108708224 : -128555600843392 : ffff8b145113ed80) 00000020:00000040:0.0:1787695882.103861:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:0.0:1787695882.103867:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000040:1.0:1787695882.103867:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b145113ed80 x1874531139538560/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695882.103874:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787695882.103879:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695882.103884:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1787695882.103885:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787695882.103891:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695882.103891:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695882.103895:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000001:0.0:1787695882.103896:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787695882.103901:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000200:1.0:1787695882.103902:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139538560, offset 0 00000008:00000001:0.0:1787695882.103906:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695882.103911:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695882.103911:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695882.103917:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000010:0.0:1787695882.103919:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b145113dc00. 00000020:00000040:0.0:1787695882.103922:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:0.0:1787695882.103927:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1787695882.103928:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 02000000:00000001:0.0:1787695882.103930:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695882.103933:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:0.0:1787695882.103936:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1787695882.103936:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 02000000:00000001:0.0:1787695882.103940:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1787695882.103941:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 02000000:00000001:0.0:1787695882.103946:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695882.103946:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695882.103951:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17017:17017:256:4294967295] 192.168.201.140@tcp LPNI seq info [17016:17016:8:4294967295] 02000000:00000010:0.0:1787695882.103955:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f5800. 00000400:00000200:1.0:1787695882.103955:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695882.103962:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695882.103964:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695882.103972:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00100000:0.0:1787695882.103974:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b145113dc00 x1874531139538944/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:1.0:1787695882.103976:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695882.103984:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695882.103989:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1787695882.103993:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695882.103995:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1787695882.103998:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695882.104001:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695882.104007:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:0.0:1787695882.104016:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1787695882.104016:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695882.104020:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000100:00000001:0.0:1787695882.104021:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695882.104026:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000040:0.0:1787695882.104029:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000002b99ca3] to pc [ptlrpcd_rcv+-1] req@ffff8b145113dc00 x1874531139538944/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.104032:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787695882.104040:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695882.104041:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b145113ed80 x1874531139538560/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787695882.104043:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787695882.104054:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787695882.104058:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787695882.104059:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1787695882.104061:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1787695882.104064:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b145113ed80 x1874531139538560/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695907 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695882.104067:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695882.104073:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695882.104074:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3769 00000100:00000001:1.0:1787695882.104077:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000200:0.0:1787695882.104084:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695882.104084:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695882.104091:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000400:00000200:0.0:1787695882.104092:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:0.0:1787695882.104098:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1787695882.104100:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:0.0:1787695882.104104:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695882.104105:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298443718 00000100:00000001:0.0:1787695882.104108:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695882.104110:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695882.104114:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:0.0:1787695882.104118:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695882.104118:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298453816 00000020:00000040:0.0:1787695882.104121:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000100:00000001:1.0:1787695882.104124:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298443718 00000020:00000001:0.0:1787695882.104125:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1787695882.104130:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000100:00000001:1.0:1787695882.104130:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298453816 00000020:00000001:0.0:1787695882.104136:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:1.0:1787695882.104137:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1787695882.104143:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:1.0:1787695882.104144:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695882.104147:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:0.0:1787695882.104152:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695882.104157:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1787695882.104157:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695882/real 1787695882] req@ffff8b145113ed80 x1874531139538560/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695882.104164:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:0.0:1787695882.104169:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000001:1.0:1787695882.104175:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695882.104178:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000040:0.0:1787695882.104179:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000400:00000010:1.0:1787695882.104182:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000100:00080000:0.0:1787695882.104185:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:0.0:1787695882.104188:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695882.104188:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 02000000:00000001:0.0:1787695882.104192:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:1.0:1787695882.104193:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 02000000:00000001:0.0:1787695882.104196:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695882.104198:0:2402:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:0.0:1787695882.104199:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695882.104203:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787695882.104205:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000200:1.0:1787695882.104207:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b145113ed80 x1874531139538560/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1787695882.104208:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695882.104212:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1787695882.104216:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b145113f800. 00000020:00000040:0.0:1787695882.104220:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:0.0:1787695882.104226:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787695882.104229:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695882.104230:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000200:1.0:1787695882.104230:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b145113ed80 x1874531139538560/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787695882.104233:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787695882.104235:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695882.104243:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695882.104245:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.104249:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695882.104250:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:0.0:1787695882.104253:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f5000. 00000100:00100000:1.0:1787695882.104255:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b145113ed80 x1874531139538560/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695882.104258:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695882.104269:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787695882.104270:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b145113f800 x1874531139539072/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1787695882.104279:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b145113ed80 x1874531139538560/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695882.104284:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787695882.104289:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695882.104295:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:0.0:1787695882.104300:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005cf982f0] to pc [ptlrpcd_rcv+-1] req@ffff8b145113f800 x1874531139539072/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.104300:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.104311:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695882.104313:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695882.104317:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695882.104317:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695882.104320:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787695882.104323:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695882.104326:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787695882.104329:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1787695882.104333:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:0.0:1787695882.104334:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787695882.104337:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:0.0:1787695882.104339:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1787695882.104343:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b145113ed80 x1874531139538560/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695882.104345:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:1.0:1787695882.104350:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b145113ed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139538560:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695882.104354:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1787695882.104358:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3769 00000100:00000040:1.0:1787695882.104359:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b145113fb80 x1874531139538688/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.104382:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695882.104385:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1787695882.104388:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:1.0:1787695882.104388:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695882.104390:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1787695882.104395:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 02000000:00000001:1.0:1787695882.104395:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787695882.104400:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787695882.104403:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b145113fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139538688:192.168.201.140@tcp:38:kworker.0 00000100:00000040:0.0:1787695882.104407:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695882.104409:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787695882.104410:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787695882.104412:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:0.0:1787695882.104414:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695882.104416:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695882.104417:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:0.0:1787695882.104422:0:38:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1787695882.104422:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:0.0:1787695882.104426:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000020:00000001:0.0:1787695882.104427:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:0.0:1787695882.104429:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 02000000:00000010:1.0:1787695882.104429:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544c00. 00000020:00000001:0.0:1787695882.104433:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 02000000:00000001:1.0:1787695882.104435:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695882.104436:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:0.0:1787695882.104442:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695882.104447:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695882.104448:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1787695882.104453:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000010:1.0:1787695882.104454:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000100:00000001:0.0:1787695882.104467:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:0.0:1787695882.104471:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000400:00000010:1.0:1787695882.104476:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00080000:0.0:1787695882.104477:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000200:1.0:1787695882.104481:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139538688, portal 10 00000100:00000001:0.0:1787695882.104483:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695882.104486:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1787695882.104488:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787695882.104492:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695882.104496:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615518108711808 : -128555600839808 : ffff8b145113fb80) 02000000:00000001:0.0:1787695882.104497:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695882.104503:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787695882.104505:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000040:1.0:1787695882.104508:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b145113fb80 x1874531139538688/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1787695882.104510:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695882.104515:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1787695882.104523:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b145113e300. 00000100:00000001:1.0:1787695882.104524:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:0.0:1787695882.104529:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000040:1.0:1787695882.104530:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695882.104535:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 02000000:00000001:0.0:1787695882.104536:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000200:1.0:1787695882.104540:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139538688, offset 0 02000000:00000001:0.0:1787695882.104546:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695882.104547:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:0.0:1787695882.104551:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787695882.104556:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1787695882.104556:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:0.0:1787695882.104560:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695882.104563:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 02000000:00000010:0.0:1787695882.104568:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b14464f7400. 00000400:00000200:1.0:1787695882.104568:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695882.104572:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:0.0:1787695882.104576:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695882.104577:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695882.104583:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00100000:0.0:1787695882.104587:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b145113e300 x1874531139539200/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695882.104591:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17018:17018:256:4294967295] 192.168.201.140@tcp LPNI seq info [17017:17017:8:4294967295] 00000400:00000200:1.0:1787695882.104602:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:0.0:1787695882.104609:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1787695882.104613:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:0.0:1787695882.104615:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695882.104621:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000040:0.0:1787695882.104626:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a6ea814f] to pc [ptlrpcd_rcv+-1] req@ffff8b145113e300 x1874531139539200/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:1.0:1787695882.104627:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695882.104638:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695882.104644:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000100:00000001:0.0:1787695882.104646:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1787695882.104651:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1787695882.104654:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:0.0:1787695882.104659:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000200:1.1:1787695882.104660:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000100:00000040:0.0:1787695882.104666:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787695882.104669:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00000040:0.0:1787695882.104674:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000010:1.0:1787695882.104675:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695882.104681:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:0.0:1787695882.104683:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3779) 00000400:00000200:1.0:1787695882.104687:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695882.104691:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695882.104694:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695882.104697:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695882.104701:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b145113fb80 x1874531139538688/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.104711:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695882.104724:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b145113fb80 x1874531139538688/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695907 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.104731:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695882.104733:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695882.104738:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695882.104742:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787695882.104745:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695882.104748:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298442567 00000100:00000001:1.0:1787695882.104750:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298453816 00000100:00000001:1.0:1787695882.104752:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298442567 00000100:00000001:1.0:1787695882.104754:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298453816 00000100:00000001:1.0:1787695882.104757:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695882.104759:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695882.104761:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695882.104765:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695882/real 1787695882] req@ffff8b145113fb80 x1874531139538688/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695882.104773:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695882.104775:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695882.104777:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695882.104780:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695882.104781:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695882.104784:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695882.104787:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b145113fb80 x1874531139538688/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695882.104796:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b145113fb80 x1874531139538688/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.104804:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695882.104805:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.104807:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695882.104809:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695882.104819:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b145113fb80 x1874531139538688/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.104826:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695882.104831:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b145113fb80 x1874531139538688/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.104838:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695882.104840:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.104842:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695882.104843:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.104845:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695882.104847:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695882.104849:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695882.104852:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695882.104854:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695882.104858:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b145113fb80 x1874531139538688/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695882.104866:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b145113fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139538688:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695882.104870:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695882.104873:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b145113c000 x1874531139538816/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.104879:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695882.104880:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695882.104882:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695882.104884:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695882.104886:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695882.104888:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b145113c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139538816:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695882.104891:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695882.104898:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695882.104900:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695882.104902:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695882.104905:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695882.104907:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695882.104912:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695882.104914:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695882.104918:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695882.104922:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139538816, portal 4 00000100:00000001:1.0:1787695882.104923:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695882.104925:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615518108696576 : -128555600855040 : ffff8b145113c000) 00000100:00000040:1.0:1787695882.104931:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b145113c000 x1874531139538816/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.104937:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695882.104939:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695882.104942:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695882.104944:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139538816, offset 0 00000400:00000200:1.0:1787695882.104948:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695882.104953:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695882.104957:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695882.104960:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695882.104962:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695882.104965:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695882.104967:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695882.104970:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17019:17019:256:4294967295] 192.168.201.140@tcp LPNI seq info [17018:17018:8:4294967295] 00000400:00000200:1.0:1787695882.104974:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695882.104980:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695882.104985:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695882.104988:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695882.104992:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695882.104995:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1787695882.104999:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695882.105002:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1787695882.105006:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695882.105013:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695882.105017:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695882.105021:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695882.105024:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695882.105026:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695882.105029:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695882.105033:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b145113c000 x1874531139538816/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105041:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695882.105044:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b145113c000 x1874531139538816/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695907 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105051:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695882.105052:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695882.105056:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695882.105059:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695882.105062:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695882.105064:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298442597 00000100:00000001:1.0:1787695882.105067:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298453816 00000100:00000001:1.0:1787695882.105069:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298442597 00000100:00000001:1.0:1787695882.105071:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298453816 00000100:00000001:1.0:1787695882.105073:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695882.105075:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695882.105077:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695882.105080:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695882/real 1787695882] req@ffff8b145113c000 x1874531139538816/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695882.105089:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695882.105090:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695882.105091:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695882.105094:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695882.105095:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695882.105103:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695882.105107:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b145113c000 x1874531139538816/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695882.105116:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b145113c000 x1874531139538816/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105123:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695882.105124:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.105126:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695882.105128:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695882.105132:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b145113c000 x1874531139538816/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105138:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695882.105143:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b145113c000 x1874531139538816/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105150:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695882.105153:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.105154:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695882.105156:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.105158:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695882.105159:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695882.105162:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695882.105166:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695882.105168:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695882.105173:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b145113c000 x1874531139538816/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695882.105182:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b145113c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139538816:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695882.105186:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.105188:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695882.105196:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b145113ed80 x1874531139538560/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105204:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695882.105205:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695882.105207:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545000. 02000000:00000001:1.0:1787695882.105210:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695882.105212:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695882.105213:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695882.105215:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695882.105217:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f4400. 02000000:00000001:1.0:1787695882.105220:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695882.105222:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695882.105223:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b145113ed80. 00000100:00000001:1.0:1787695882.105226:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695882.105228:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.105230:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695882.105233:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b145113fb80 x1874531139538688/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105240:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695882.105241:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695882.105243:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544c00. 02000000:00000001:1.0:1787695882.105246:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695882.105247:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695882.105248:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695882.105250:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695882.105252:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f5400. 02000000:00000001:1.0:1787695882.105254:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695882.105255:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695882.105257:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b145113fb80. 00000100:00000001:1.0:1787695882.105259:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695882.105260:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.105262:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695882.105265:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b145113c000 x1874531139538816/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105271:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695882.105276:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695882.105278:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695882.105280:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695882.105282:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695882.105283:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695882.105285:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695882.105286:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f5c00. 02000000:00000001:1.0:1787695882.105288:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695882.105289:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695882.105291:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b145113c000. 00000100:00000001:1.0:1787695882.105293:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695882.105294:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.105296:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.105299:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695882.105301:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695882.105303:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695882.105304:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695882.105306:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695882.105310:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b145113dc00 x1874531139538944/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105317:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695882.105318:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695882.105321:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695882.105322:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695882.105325:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695882.105328:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b145113dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139538944:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695882.105331:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695882.105333:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695882.105335:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695882.105337:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695882.105340:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695882.105342:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695882.105345:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695882.105348:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695882.105351:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695882.105354:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139538944, portal 4 00000100:00000001:1.0:1787695882.105379:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695882.105383:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615518108703744 : -128555600847872 : ffff8b145113dc00) 00000100:00000040:1.0:1787695882.105391:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b145113dc00 x1874531139538944/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105399:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695882.105403:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695882.105406:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695882.105409:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139538944, offset 0 00000400:00000200:1.0:1787695882.105415:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695882.105420:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695882.105424:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695882.105428:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695882.105431:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695882.105433:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695882.105436:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695882.105439:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17020:17020:256:4294967295] 192.168.201.140@tcp LPNI seq info [17019:17019:8:4294967295] 00000400:00000200:1.0:1787695882.105444:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695882.105450:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695882.105468:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695882.105472:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695882.105476:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695882.105480:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1787695882.105484:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695882.105487:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1787695882.105496:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695882.105499:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695882.105502:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695882.105506:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695882.105509:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695882.105511:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695882.105514:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695882.105518:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b145113dc00 x1874531139538944/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105537:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695882.105542:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b145113dc00 x1874531139538944/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695907 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105549:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695882.105550:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695882.105555:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695882.105558:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695882.105561:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695882.105563:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298442270 00000100:00000001:1.0:1787695882.105566:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298453816 00000100:00000001:1.0:1787695882.105568:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298442270 00000100:00000001:1.0:1787695882.105570:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298453816 00000100:00000001:1.0:1787695882.105572:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695882.105574:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695882.105576:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695882.105579:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695882/real 1787695882] req@ffff8b145113dc00 x1874531139538944/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695882.105587:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695882.105589:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695882.105590:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695882.105592:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695882.105594:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695882.105596:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695882.105599:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b145113dc00 x1874531139538944/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695882.105606:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b145113dc00 x1874531139538944/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105612:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695882.105619:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.105621:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695882.105623:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695882.105627:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b145113dc00 x1874531139538944/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105634:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695882.105639:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b145113dc00 x1874531139538944/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105645:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695882.105647:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.105649:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695882.105651:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.105653:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695882.105655:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695882.105658:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695882.105661:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695882.105663:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695882.105667:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b145113dc00 x1874531139538944/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695882.105675:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b145113dc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139538944:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695882.105679:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695882.105682:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b145113f800 x1874531139539072/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105688:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695882.105690:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695882.105692:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695882.105693:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695882.105695:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695882.105704:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b145113f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139539072:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695882.105707:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695882.105709:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695882.105711:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695882.105713:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695882.105715:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544c00. 02000000:00000001:1.0:1787695882.105717:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695882.105721:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695882.105723:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695882.105727:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695882.105730:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139539072, portal 10 00000100:00000001:1.0:1787695882.105732:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695882.105734:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615518108710912 : -128555600840704 : ffff8b145113f800) 00000100:00000040:1.0:1787695882.105740:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b145113f800 x1874531139539072/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105745:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695882.105748:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695882.105750:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695882.105752:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139539072, offset 0 00000400:00000200:1.0:1787695882.105756:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695882.105760:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695882.105764:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695882.105768:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695882.105770:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695882.105773:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695882.105775:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695882.105778:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17021:17021:256:4294967295] 192.168.201.140@tcp LPNI seq info [17020:17020:8:4294967295] 00000400:00000200:1.0:1787695882.105783:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695882.105790:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695882.105795:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695882.105799:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695882.105810:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695882.105813:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1787695882.105817:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695882.105820:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1787695882.105823:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695882.105826:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695882.105829:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695882.105833:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695882.105836:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695882.105838:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695882.105841:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695882.105845:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b145113f800 x1874531139539072/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105853:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695882.105856:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b145113f800 x1874531139539072/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695907 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105863:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695882.105865:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695882.105869:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695882.105872:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695882.105875:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695882.105878:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298442456 00000100:00000001:1.0:1787695882.105881:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298453816 00000100:00000001:1.0:1787695882.105883:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298442456 00000100:00000001:1.0:1787695882.105885:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298453816 00000100:00000001:1.0:1787695882.105887:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695882.105890:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695882.105892:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695882.105896:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695882/real 1787695882] req@ffff8b145113f800 x1874531139539072/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695882.105905:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695882.105912:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695882.105914:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695882.105916:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695882.105918:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695882.105921:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695882.105925:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b145113f800 x1874531139539072/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695882.105934:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b145113f800 x1874531139539072/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105941:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695882.105943:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.105945:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695882.105946:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695882.105950:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b145113f800 x1874531139539072/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105957:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695882.105961:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b145113f800 x1874531139539072/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.105969:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695882.105971:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.105972:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695882.105974:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.105976:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695882.105977:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695882.105979:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695882.105982:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695882.105984:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695882.105988:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b145113f800 x1874531139539072/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695882.106008:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b145113f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139539072:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695882.106012:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695882.106016:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b145113e300 x1874531139539200/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.106022:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695882.106024:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695882.106026:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695882.106028:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695882.106030:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695882.106033:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b145113e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139539200:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695882.106035:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695882.106037:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695882.106039:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695882.106041:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695882.106044:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545000. 02000000:00000001:1.0:1787695882.106046:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695882.106049:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695882.106052:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695882.106055:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695882.106058:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139539200, portal 10 00000100:00000001:1.0:1787695882.106060:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695882.106061:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615518108705536 : -128555600846080 : ffff8b145113e300) 00000100:00000040:1.0:1787695882.106067:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b145113e300 x1874531139539200/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.106072:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695882.106073:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695882.106075:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695882.106076:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139539200, offset 0 00000400:00000200:1.0:1787695882.106079:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695882.106086:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695882.106089:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695882.106091:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695882.106094:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695882.106097:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695882.106099:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695882.106102:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17022:17022:256:4294967295] 192.168.201.140@tcp LPNI seq info [17021:17021:8:4294967295] 00000400:00000200:1.0:1787695882.106106:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695882.106112:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695882.106117:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695882.106121:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695882.106125:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695882.106128:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1787695882.106132:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695882.106135:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1787695882.106139:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695882.106141:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695882.106144:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695882.106148:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695882.106151:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695882.106153:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695882.106155:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695882.106160:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b145113e300 x1874531139539200/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695907 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.106167:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695882.106170:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b145113e300 x1874531139539200/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695907 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.106177:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695882.106179:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695882.106182:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695882.106185:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695882.106189:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695882.106197:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298442434 00000100:00000001:1.0:1787695882.106200:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298453816 00000100:00000001:1.0:1787695882.106202:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298442434 00000100:00000001:1.0:1787695882.106203:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298453816 00000100:00000001:1.0:1787695882.106204:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695882.106206:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695882.106207:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695882.106211:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695882/real 1787695882] req@ffff8b145113e300 x1874531139539200/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695882.106216:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695882.106218:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695882.106219:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695882.106221:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695882.106222:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695882.106223:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695882.106226:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b145113e300 x1874531139539200/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695882.106231:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b145113e300 x1874531139539200/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.106235:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695882.106236:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.106237:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695882.106238:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695882.106240:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b145113e300 x1874531139539200/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.106244:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695882.106247:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b145113e300 x1874531139539200/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.106252:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695882.106256:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.106257:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695882.106258:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.106260:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695882.106261:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695882.106263:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695882.106265:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695882.106266:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695882.106268:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b145113e300 x1874531139539200/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695882.106273:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b145113e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139539200:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695882.106276:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.106278:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695882.106280:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b145113dc00 x1874531139538944/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.106284:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695882.106285:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695882.106286:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695882.106288:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695882.106289:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695882.106290:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695882.106291:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695882.106292:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f5800. 02000000:00000001:1.0:1787695882.106294:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695882.106295:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695882.106296:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b145113dc00. 00000100:00000001:1.0:1787695882.106298:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695882.106299:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.106300:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695882.106302:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b145113f800 x1874531139539072/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.106316:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695882.106318:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695882.106319:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544c00. 02000000:00000001:1.0:1787695882.106321:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695882.106323:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695882.106324:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695882.106326:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695882.106328:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f5000. 02000000:00000001:1.0:1787695882.106330:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695882.106331:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695882.106332:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b145113f800. 00000100:00000001:1.0:1787695882.106335:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695882.106336:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.106338:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695882.106342:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b145113e300 x1874531139539200/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695907 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695882.106349:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695882.106351:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695882.106352:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545000. 02000000:00000001:1.0:1787695882.106355:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695882.106356:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695882.106357:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695882.106359:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695882.106360:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b14464f7400. 02000000:00000001:1.0:1787695882.106384:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695882.106387:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695882.106388:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b145113e300. 00000100:00000001:1.0:1787695882.106391:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695882.106393:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.106394:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695882.106398:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695882.106400:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695882.106402:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695882.106403:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695882.166496:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695882.166503:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695882.166505:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695882.166510:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695882.166514:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695882.166518:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695882.166521:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695882.166523:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695882.166525:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695882.806493:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695882.806514:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695882.806519:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695882.806520:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695882.806523:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695882.806524:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695882.806526:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695882.806527:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695882.806530:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695882.806531:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695882.806533:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695882.806534:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695882.806605:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695882.806610:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695882.806613:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695883.190419:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695883.190425:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695883.190427:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695883.190431:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695883.190434:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695883.190450:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695883.190453:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695883.190455:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695883.190457:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695883.830473:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695883.830480:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695883.830482:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695883.830484:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695883.830488:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695883.830490:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695883.830492:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695883.830493:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695883.830496:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695883.830497:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695883.830499:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695883.830500:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695883.830501:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695883.830502:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695883.830503:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695883.830504:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695883.830507:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695883.830508:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695883.830510:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695883.830511:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695883.830513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695883.830514:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695883.830516:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695883.830517:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695883.830579:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695883.830584:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695883.830587:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695884.214447:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695884.214454:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695884.214456:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695884.214460:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695884.214463:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695884.214466:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695884.214468:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695884.214470:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695884.214472:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695884.854451:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695884.854457:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695884.854461:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695884.854479:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695884.854482:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695884.854483:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695884.854485:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695884.854486:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695884.854489:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695884.854490:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695884.854492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695884.854492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695884.854542:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695884.854545:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695884.854548:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695885.238475:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695885.238484:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695885.238486:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695885.238492:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695885.238496:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695885.238500:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695885.238503:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695885.238506:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695885.238508:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695885.878496:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695885.878504:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695885.878511:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695885.878512:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695885.878516:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695885.878518:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695885.878522:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695885.878523:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695885.878527:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695885.878528:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695885.878531:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695885.878533:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695885.878592:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695885.878595:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695885.878597:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695886.262514:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695886.262522:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695886.262524:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695886.262530:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695886.262533:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695886.262537:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695886.262540:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695886.262543:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695886.262546:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695886.902481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695886.902487:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695886.902491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695886.902492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695886.902495:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695886.902498:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695886.902500:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695886.902505:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695886.902506:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695886.902507:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695886.902510:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695886.902512:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695886.902513:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695886.902515:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695886.902517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695886.902518:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695886.902519:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695886.902520:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695886.902538:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695886.902539:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695886.902540:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695886.902541:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695886.902542:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695886.902544:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695886.902546:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695886.902547:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695886.902550:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695886.902551:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695886.902551:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695886.902553:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695886.902554:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695886.902554:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695886.902555:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695886.902556:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695886.902557:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695886.902558:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695886.902625:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695886.902631:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695886.902635:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787695887.222496:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695887.222508:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787695887.222511:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695887.222515:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695887.222520:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695887.222527:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3774 00000400:00000200:3.0:1787695887.222535:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695887.222542:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695887.222546:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695887.222549:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695887.222553:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695887.222555:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695887.222558:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695887.222562:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695887.222565:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:3.0:1787695887.222567:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695887.222588:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787695887.222591:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787695887.222594:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787695887.222598:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695887.222600:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695887.222602:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695887.222604:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695887.222607:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787695887.222610:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695887.222613:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695887.222616:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695887.222620:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695887.222623:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695887.222625:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787695887.222628:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695887.222635:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7100. 00000020:00000040:3.0:1787695887.222639:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787695887.222658:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695887.222660:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695887.222663:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695887.222665:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695887.222669:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695887.222678:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674800. 00000100:00000001:3.0:1787695887.222684:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695887.222694:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7100 x1874531139539328/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695887.222704:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695887.222708:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695887.222713:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000004e8abd7] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7100 x1874531139539328/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695887.222740:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695887.222744:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695887.222753:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695887.222755:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695887.222756:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695887.222759:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695887.222762:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3774 00000400:00000200:3.0:1787695887.222766:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695887.222769:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695887.222770:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787695887.222771:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695887.222777:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695887.222778:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695887.222779:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695887.222783:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695887.222785:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695887.222786:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695887.222787:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:3.0:1787695887.222789:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695887.222790:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1787695887.222791:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:3.0:1787695887.222795:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:3.0:1787695887.222796:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:3.0:1787695887.222801:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695887.222802:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7100 x1874531139539328/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695887.222803:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695887.222808:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695887.222812:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695887.222816:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:3.0:1787695887.222822:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695887.222829:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695887.222829:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695887.222833:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695887.222835:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695887.222840:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695887.222841:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695887.222847:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695887.222847:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695887.222852:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787695887.222854:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787695887.222858:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787695887.222861:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139539328:192.168.201.140@tcp:250:kworker.0 00000100:00000001:1.0:1787695887.222867:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:3.0:1787695887.222870:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695887.222872:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787695887.222875:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695887.222876:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695887.222881:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:3.0:1787695887.222883:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4e00. 02000000:00000010:1.0:1787695887.222886:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545000. 00000020:00000040:3.0:1787695887.222887:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:1.0:1787695887.222892:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695887.222893:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695887.222897:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695887.222899:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695887.222901:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1787695887.222903:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787695887.222904:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695887.222909:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 00000100:00000001:3.0:1787695887.222912:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695887.222918:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000100:00100000:3.0:1787695887.222919:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4e00 x1874531139539456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695887.222928:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1787695887.222930:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000001:3.0:1787695887.222931:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695887.222936:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcd77cd6] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4e00 x1874531139539456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695887.222942:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139539328, portal 25 00000100:00000001:3.0:1787695887.222944:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695887.222948:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787695887.222948:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695887.222954:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930156288 : -128555779395328 : ffff8b14466f7100) 00000100:00080000:3.0:1787695887.222954:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695887.222957:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695887.222960:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695887.222963:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:1.0:1787695887.222966:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7100 x1874531139539328/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695887.222968:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3774 00000400:00000200:3.0:1787695887.222977:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695887.222983:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695887.222986:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695887.222987:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1787695887.222990:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695887.222995:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695887.222996:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695887.223000:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000010:1.0:1787695887.223001:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695887.223006:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139539328, offset 0 00000100:00000001:3.0:1787695887.223010:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695887.223012:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695887.223014:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:3.0:1787695887.223017:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:1.0:1787695887.223018:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000020:00000040:3.0:1787695887.223019:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:3.0:1787695887.223023:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:3.0:1787695887.223024:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000400:00000200:1.0:1787695887.223029:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695887.223029:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695887.223033:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695887.223036:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695887.223040:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:3.0:1787695887.223041:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695887.223046:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000400:00000200:1.0:1787695887.223049:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:3.0:1787695887.223052:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695887.223055:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695887.223057:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 02000000:00000001:3.0:1787695887.223058:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1787695887.223063:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 02000000:00000001:3.0:1787695887.223063:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695887.223068:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695887.223070:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:3.0:1787695887.223071:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695887.223075:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1787695887.223077:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17023:17023:256:4294967295] 192.168.201.140@tcp LPNI seq info [17022:17022:8:4294967295] 00000008:00000001:3.0:1787695887.223078:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695887.223082:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695887.223086:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000010:3.0:1787695887.223087:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7480. 00000020:00000040:3.0:1787695887.223091:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695887.223093:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695887.223093:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695887.223095:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695887.223095:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695887.223097:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695887.223099:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 00000100:00000001:3.0:1787695887.223104:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695887.223107:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00100000:3.0:1787695887.223112:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7480 x1874531139539584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1787695887.223118:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695887.223126:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695887.223136:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787695887.223140:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:1.0:1787695887.223141:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000100:00000001:3.0:1787695887.223146:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1787695887.223150:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695887.223154:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000100:00000040:3.0:1787695887.223157:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043daeda0] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7480 x1874531139539584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1787695887.223160:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695887.223164:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695887.223170:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1787695887.223174:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695887.223177:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00080000:3.0:1787695887.223182:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695887.223184:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695887.223188:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00080000:3.0:1787695887.223190:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787695887.223192:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695887.223194:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695887.223199:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787695887.223200:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695887.223205:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3774 00000100:00000200:1.0:1787695887.223208:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7100 x1874531139539328/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695887.223214:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695887.223221:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695887.223225:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695887.223226:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695887.223229:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695887.223233:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7100 x1874531139539328/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695912 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695887.223234:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695887.223239:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695887.223244:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695887.223248:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695887.223252:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695887.223252:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:3.0:1787695887.223254:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695887.223256:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:3.0:1787695887.223257:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:3.0:1787695887.223260:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:3.0:1787695887.223262:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:1.0:1787695887.223263:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695887.223265:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695887.223268:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695887.223271:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695887.223273:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695887.223277:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695887.223278:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298458936 00000100:00000001:1.0:1787695887.223284:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298453816 00000100:00000001:1.0:1787695887.223290:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298458936 00000100:00000001:1.0:1787695887.223292:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298453816 00000020:00000040:3.0:1787695887.223293:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00000001:1.0:1787695887.223294:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695887.223299:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695887.223302:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695887.223303:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787695887.223306:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695887.223308:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695887.223310:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695887.223312:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695887.223315:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695887.223317:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00100000:1.0:1787695887.223321:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695887/real 1787695887] req@ffff8b14466f7100 x1874531139539328/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1787695887.223322:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695887.223325:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695887.223332:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7800. 00000400:00000001:1.0:1787695887.223335:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000040:3.0:1787695887.223337:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000400:00000001:1.0:1787695887.223338:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695887.223342:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 02000000:00000001:3.0:1787695887.223344:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1787695887.223346:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 02000000:00000001:3.0:1787695887.223347:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695887.223350:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:3.0:1787695887.223353:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787695887.223354:0:2402:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1787695887.223358:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:1.0:1787695887.223359:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7100 x1874531139539328/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695887.223397:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695887.223403:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 00000100:00000001:3.0:1787695887.223406:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695887.223415:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7800 x1874531139539712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1787695887.223419:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7100 x1874531139539328/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695887.223432:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695887.223436:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695887.223443:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:3.0:1787695887.223444:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7c57981] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7800 x1874531139539712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695887.223450:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695887.223454:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695887.223455:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695887.223459:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1787695887.223462:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695887.223462:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695887.223467:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695887.223468:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695887.223470:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:1.0:1787695887.223477:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7100 x1874531139539328/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695887.223480:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3774 00000400:00000200:3.0:1787695887.223488:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695887.223494:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695887.223495:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695887.223496:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695887.223502:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695887.223503:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695887.223505:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7100 x1874531139539328/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695887.223509:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695887.223513:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695887.223518:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695887.223520:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000100:00000001:1.0:1787695887.223523:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1787695887.223523:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695887.223527:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000100:00000001:1.0:1787695887.223528:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787695887.223530:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000001:1.0:1787695887.223531:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1787695887.223534:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:3.0:1787695887.223538:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695887.223539:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695887.223540:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695887.223541:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695887.223542:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1787695887.223543:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:3.0:1787695887.223547:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695887.223550:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787695887.223550:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695887.223554:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1787695887.223555:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695887.223559:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1787695887.223560:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:3.0:1787695887.223563:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695887.223566:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00080000:1.0:1787695887.223570:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000008:00100000:3.0:1787695887.223570:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787695887.223574:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695887.223577:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695887.223578:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000010:3.0:1787695887.223580:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7b80. 00000020:00000040:3.0:1787695887.223584:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:3.0:1787695887.223586:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000040:1.0:1787695887.223589:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7100 x1874531139539328/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695887.223592:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695887.223597:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695887.223599:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695887.223601:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695887.223606:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676000. 00000100:00000001:3.0:1787695887.223611:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695887.223615:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139539328:192.168.201.140@tcp:250:kworker.0 00000100:00100000:3.0:1787695887.223619:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7b80 x1874531139539840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695887.223622:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.223627:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787695887.223629:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695887.223633:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695887.223634:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7100 x1874531139539328/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695887.223640:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b87e1da] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7b80 x1874531139539840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.223647:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695887.223651:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1787695887.223653:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695887.223654:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545000. 00000100:00080000:3.0:1787695887.223657:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1787695887.223660:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:3.0:1787695887.223660:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695887.223664:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000020:00000001:1.0:1787695887.223665:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000100:00080000:3.0:1787695887.223666:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695887.223668:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695887.223670:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3774 00000020:00000040:1.0:1787695887.223674:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000400:00000200:3.0:1787695887.223675:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:1.0:1787695887.223680:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000400:00000200:3.0:1787695887.223681:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 02000000:00000010:1.0:1787695887.223684:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674800. 00000100:00080000:3.0:1787695887.223685:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695887.223690:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695887.223692:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1787695887.223693:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787695887.223696:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:3.0:1787695887.223697:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000010:1.0:1787695887.223699:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7100. 00000100:00000001:3.0:1787695887.223700:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695887.223703:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000020:00000001:3.0:1787695887.223704:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695887.223705:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1787695887.223706:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000100:00000001:1.0:1787695887.223709:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787695887.223710:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695887.223713:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000020:00000001:3.0:1787695887.223715:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:1.0:1787695887.223718:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695887.223721:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695887.223726:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:1.0:1787695887.223727:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695887.223731:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1787695887.223732:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695887.223733:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787695887.223736:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695887.223742:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695887.223743:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4e00 x1874531139539456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695887.223744:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695887.223747:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:3.0:1787695887.223752:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695887.223755:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695887.223759:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695887.223762:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695887.223766:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695887.223767:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695887.223768:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695887.223772:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787695887.223772:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695887.223778:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695887.223780:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787695887.223781:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695887.223785:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1787695887.223786:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6a00. 00000020:00000040:3.0:1787695887.223789:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:1.0:1787695887.223793:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695887.223793:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695887.223796:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695887.223799:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:1.0:1787695887.223800:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139539456:192.168.201.140@tcp:38:kworker.0 02000000:00000001:3.0:1787695887.223801:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695887.223804:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695887.223810:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000010:3.0:1787695887.223811:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 02000000:00000001:1.0:1787695887.223815:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787695887.223816:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695887.223820:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695887.223825:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6a00 x1874531139539968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787695887.223826:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695887.223832:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545000. 02000000:00000001:1.0:1787695887.223836:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695887.223839:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695887.223844:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695887.223845:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695887.223851:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000100:00000040:3.0:1787695887.223852:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002ad47e8f] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6a00 x1874531139539968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1787695887.223858:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695887.223865:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139539456, portal 10 00000100:00000001:1.0:1787695887.223868:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787695887.223869:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695887.223870:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930147328 : -128555779404288 : ffff8b14466f4e00) 00000100:00000040:1.0:1787695887.223878:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4e00 x1874531139539456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695887.223879:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695887.223882:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695887.223883:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695887.223886:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695887.223891:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695887.223896:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3774 00000100:00000001:1.0:1787695887.223897:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1787695887.223903:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:1.0:1787695887.223906:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000200:3.0:1787695887.223909:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000010:1.0:1787695887.223913:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00080000:3.0:1787695887.223913:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695887.223918:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000200:1.0:1787695887.223921:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139539456, offset 0 00000100:00000001:3.0:1787695887.223922:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695887.223925:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695887.223929:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695887.223931:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695887.223932:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000020:00000001:3.0:1787695887.223935:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:1.0:1787695887.223936:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000020:00000040:3.0:1787695887.223937:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:3.0:1787695887.223941:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000020:00000001:3.0:1787695887.223943:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000400:00000200:1.0:1787695887.223948:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695887.223949:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695887.223953:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695887.223957:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695887.223960:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:3.0:1787695887.223960:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695887.223966:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000400:00000200:1.0:1787695887.223967:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:3.0:1787695887.223971:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:1.0:1787695887.223974:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:3.0:1787695887.223976:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695887.223979:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 02000000:00000001:3.0:1787695887.223980:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1787695887.223985:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 02000000:00000001:3.0:1787695887.223985:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695887.223990:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695887.223991:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17024:17024:256:4294967295] 192.168.201.140@tcp LPNI seq info [17023:17023:8:4294967295] 00000100:00000001:3.0:1787695887.223993:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:1.0:1787695887.223997:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000008:00100000:3.0:1787695887.224001:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695887.224007:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695887.224010:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695887.224011:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000010:3.0:1787695887.224015:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5c00. 00000800:00000200:1.0:1787695887.224020:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000020:00000040:3.0:1787695887.224021:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:3.0:1787695887.224027:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000010:1.0:1787695887.224028:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 02000000:00000001:3.0:1787695887.224028:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695887.224030:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695887.224033:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695887.224035:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695887.224037:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695887.224041:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 02000000:00000010:3.0:1787695887.224041:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675400. 00000100:00000001:3.0:1787695887.224048:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1787695887.224049:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695887.224053:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000100:00100000:3.0:1787695887.224056:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5c00 x1874531139540096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787695887.224061:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695887.224066:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695887.224072:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1787695887.224072:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695887.224077:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695887.224080:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00000040:3.0:1787695887.224083:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005bfbcee1] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5c00 x1874531139540096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695887.224085:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695887.224089:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695887.224095:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787695887.224096:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695887.224099:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1787695887.224102:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4e00 x1874531139539456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695887.224103:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695887.224107:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787695887.224114:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695887.224120:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3785) 00000100:00000001:1.0:1787695887.224121:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695887.224127:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4e00 x1874531139539456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695912 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224142:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695887.224143:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695887.224147:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695887.224150:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000400:00000200:1.0:1787695887.224153:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695887.224154:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298448838 00000100:00000001:1.0:1787695887.224157:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298458936 00000100:00000001:1.0:1787695887.224158:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298448838 00000100:00000001:1.0:1787695887.224160:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298458936 00000100:00000001:1.0:1787695887.224161:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695887.224163:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695887.224164:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695887.224167:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695887/real 1787695887] req@ffff8b14466f4e00 x1874531139539456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695887.224172:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695887.224173:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695887.224174:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695887.224176:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695887.224177:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695887.224179:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695887.224181:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4e00 x1874531139539456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695887.224186:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4e00 x1874531139539456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224191:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695887.224192:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.224193:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695887.224194:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695887.224196:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4e00 x1874531139539456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224205:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695887.224208:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4e00 x1874531139539456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224212:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695887.224214:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.224215:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695887.224216:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.224217:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695887.224218:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695887.224220:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695887.224222:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695887.224223:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695887.224225:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4e00 x1874531139539456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695887.224231:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139539456:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695887.224233:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695887.224236:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7480 x1874531139539584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224240:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695887.224241:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695887.224242:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695887.224243:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695887.224244:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695887.224246:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139539584:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695887.224248:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695887.224249:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695887.224253:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695887.224255:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695887.224257:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544c00. 02000000:00000001:1.0:1787695887.224258:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695887.224261:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695887.224263:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695887.224265:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695887.224267:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139539584, portal 10 00000100:00000001:1.0:1787695887.224269:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695887.224270:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930157184 : -128555779394432 : ffff8b14466f7480) 00000100:00000040:1.0:1787695887.224274:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7480 x1874531139539584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224278:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695887.224280:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695887.224281:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695887.224283:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139539584, offset 0 00000400:00000200:1.0:1787695887.224286:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695887.224288:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695887.224291:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695887.224293:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695887.224294:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695887.224296:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695887.224298:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695887.224300:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17025:17025:256:4294967295] 192.168.201.140@tcp LPNI seq info [17024:17024:8:4294967295] 00000400:00000200:1.0:1787695887.224303:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695887.224307:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695887.224310:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695887.224313:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695887.224315:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695887.224317:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1787695887.224320:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695887.224322:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000800:00000100:1.0:1787695887.224325:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695887.224329:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695887.224331:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695887.224333:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695887.224335:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695887.224337:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695887.224338:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695887.224341:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7480 x1874531139539584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224346:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695887.224348:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7480 x1874531139539584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695912 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224352:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695887.224353:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695887.224356:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695887.224358:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787695887.224360:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695887.224361:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298447686 00000100:00000001:1.0:1787695887.224391:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298458936 00000100:00000001:1.0:1787695887.224394:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298447687 00000100:00000001:1.0:1787695887.224396:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298458936 00000100:00000001:1.0:1787695887.224397:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695887.224399:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695887.224400:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695887.224404:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695887/real 1787695887] req@ffff8b14466f7480 x1874531139539584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695887.224410:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695887.224411:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695887.224412:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695887.224414:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695887.224415:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695887.224421:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695887.224425:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7480 x1874531139539584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695887.224434:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7480 x1874531139539584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224440:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695887.224442:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.224444:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695887.224445:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695887.224449:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7480 x1874531139539584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224457:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695887.224461:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7480 x1874531139539584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224470:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695887.224472:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.224474:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695887.224475:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.224477:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695887.224479:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695887.224481:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695887.224483:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695887.224485:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695887.224488:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7480 x1874531139539584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695887.224493:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139539584:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695887.224496:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695887.224503:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7800 x1874531139539712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224507:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695887.224509:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695887.224510:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695887.224511:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695887.224513:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695887.224515:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139539712:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695887.224516:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695887.224518:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695887.224519:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695887.224520:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695887.224522:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695887.224524:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695887.224526:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695887.224528:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695887.224531:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695887.224533:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139539712, portal 4 00000100:00000001:1.0:1787695887.224534:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695887.224535:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930158080 : -128555779393536 : ffff8b14466f7800) 00000100:00000040:1.0:1787695887.224539:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7800 x1874531139539712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224544:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695887.224545:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695887.224546:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695887.224548:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139539712, offset 0 00000400:00000200:1.0:1787695887.224551:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695887.224553:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695887.224556:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695887.224558:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695887.224560:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695887.224562:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695887.224565:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695887.224573:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17026:17026:256:4294967295] 192.168.201.140@tcp LPNI seq info [17025:17025:8:4294967295] 00000400:00000200:1.0:1787695887.224577:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695887.224585:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695887.224590:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695887.224594:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695887.224598:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695887.224601:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1787695887.224605:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695887.224608:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000800:00000100:1.0:1787695887.224611:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695887.224612:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695887.224614:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695887.224617:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695887.224619:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695887.224621:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695887.224623:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695887.224625:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7800 x1874531139539712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224631:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695887.224633:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7800 x1874531139539712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695912 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224637:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695887.224638:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695887.224641:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695887.224643:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695887.224645:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695887.224647:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298447717 00000100:00000001:1.0:1787695887.224649:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298458936 00000100:00000001:1.0:1787695887.224650:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298447717 00000100:00000001:1.0:1787695887.224652:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298458936 00000100:00000001:1.0:1787695887.224653:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695887.224658:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695887.224660:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695887.224663:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695887/real 1787695887] req@ffff8b14466f7800 x1874531139539712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695887.224669:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695887.224670:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695887.224671:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695887.224672:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695887.224673:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695887.224675:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695887.224677:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7800 x1874531139539712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695887.224682:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7800 x1874531139539712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224686:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695887.224687:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.224688:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695887.224689:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695887.224691:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7800 x1874531139539712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224695:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695887.224697:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7800 x1874531139539712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224701:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695887.224703:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.224704:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695887.224705:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.224706:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695887.224707:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695887.224709:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695887.224715:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695887.224716:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695887.224719:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7800 x1874531139539712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695887.224724:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139539712:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695887.224727:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695887.224729:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7b80 x1874531139539840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224733:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695887.224734:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695887.224735:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695887.224736:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695887.224737:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695887.224740:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139539840:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695887.224742:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695887.224744:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695887.224745:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695887.224747:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695887.224750:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695887.224752:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695887.224756:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695887.224758:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695887.224761:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695887.224764:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139539840, portal 4 00000100:00000001:1.0:1787695887.224767:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695887.224768:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930158976 : -128555779392640 : ffff8b14466f7b80) 00000100:00000040:1.0:1787695887.224775:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7b80 x1874531139539840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224782:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695887.224789:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695887.224792:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695887.224795:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139539840, offset 0 00000400:00000200:1.0:1787695887.224798:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695887.224801:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695887.224804:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695887.224806:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695887.224808:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695887.224810:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695887.224813:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695887.224816:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17027:17027:256:4294967295] 192.168.201.140@tcp LPNI seq info [17026:17026:8:4294967295] 00000400:00000200:1.0:1787695887.224820:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695887.224827:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695887.224832:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695887.224835:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695887.224839:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695887.224842:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1787695887.224846:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695887.224849:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000800:00000100:1.0:1787695887.224853:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695887.224856:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695887.224859:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695887.224862:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695887.224864:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695887.224865:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695887.224868:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695887.224870:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7b80 x1874531139539840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224876:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695887.224878:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7b80 x1874531139539840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695912 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224882:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695887.224883:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695887.224890:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695887.224893:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695887.224895:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695887.224897:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298447389 00000100:00000001:1.0:1787695887.224898:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298458936 00000100:00000001:1.0:1787695887.224900:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298447389 00000100:00000001:1.0:1787695887.224902:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298458936 00000100:00000001:1.0:1787695887.224904:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695887.224907:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695887.224908:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695887.224912:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695887/real 1787695887] req@ffff8b14466f7b80 x1874531139539840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695887.224921:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695887.224923:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695887.224925:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695887.224928:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695887.224930:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695887.224932:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695887.224936:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7b80 x1874531139539840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695887.224944:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7b80 x1874531139539840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224951:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695887.224952:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.224954:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695887.224955:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695887.224957:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7b80 x1874531139539840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224962:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695887.224964:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7b80 x1874531139539840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.224974:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695887.224976:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.224977:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695887.224978:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.224979:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695887.224980:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695887.224982:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695887.224984:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695887.224985:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695887.224988:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7b80 x1874531139539840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695887.224993:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139539840:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695887.224995:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.224996:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695887.224998:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4e00 x1874531139539456/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.225002:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695887.225003:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695887.225005:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545000. 02000000:00000001:1.0:1787695887.225007:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695887.225008:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695887.225009:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695887.225010:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695887.225012:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:1.0:1787695887.225013:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695887.225014:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695887.225016:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4e00. 00000100:00000001:1.0:1787695887.225018:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695887.225018:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.225022:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695887.225025:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7480 x1874531139539584/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.225029:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695887.225030:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695887.225031:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544c00. 02000000:00000001:1.0:1787695887.225033:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695887.225034:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695887.225035:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695887.225036:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695887.225037:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:1.0:1787695887.225039:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695887.225040:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695887.225041:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7480. 00000100:00000001:1.0:1787695887.225042:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695887.225043:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.225044:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695887.225046:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7800 x1874531139539712/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.225050:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695887.225050:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695887.225051:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695887.225052:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695887.225053:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695887.225054:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695887.225055:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695887.225056:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:1.0:1787695887.225058:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695887.225059:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695887.225060:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7800. 00000100:00000001:1.0:1787695887.225061:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695887.225062:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.225063:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695887.225064:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7b80 x1874531139539840/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.225074:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695887.225075:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695887.225076:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695887.225078:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695887.225078:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695887.225079:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695887.225080:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695887.225081:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676000. 02000000:00000001:1.0:1787695887.225082:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695887.225083:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695887.225084:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7b80. 00000100:00000001:1.0:1787695887.225085:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695887.225086:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.225087:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.225090:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695887.225091:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695887.225092:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695887.225093:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695887.225094:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695887.225096:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6a00 x1874531139539968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.225100:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695887.225101:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695887.225102:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695887.225104:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695887.225105:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695887.225107:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139539968:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695887.225109:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695887.225110:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695887.225111:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695887.225112:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695887.225114:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695887.225116:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695887.225121:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695887.225123:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695887.225125:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695887.225128:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139539968, portal 10 00000100:00000001:1.0:1787695887.225129:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695887.225130:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930154496 : -128555779397120 : ffff8b14466f6a00) 00000100:00000040:1.0:1787695887.225134:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6a00 x1874531139539968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.225138:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695887.225140:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695887.225141:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695887.225143:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139539968, offset 0 00000400:00000200:1.0:1787695887.225146:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695887.225148:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695887.225150:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695887.225152:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695887.225154:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695887.225156:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695887.225158:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695887.225160:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17028:17028:256:4294967295] 192.168.201.140@tcp LPNI seq info [17027:17027:8:4294967295] 00000400:00000200:1.0:1787695887.225163:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695887.225167:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695887.225170:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695887.225172:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695887.225174:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695887.225176:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1787695887.225179:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695887.225181:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000800:00000100:1.0:1787695887.225183:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695887.225185:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695887.225187:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695887.225189:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695887.225191:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695887.225197:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695887.225199:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695887.225201:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6a00 x1874531139539968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.225206:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695887.225208:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6a00 x1874531139539968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695912 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.225213:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695887.225214:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695887.225216:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695887.225218:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695887.225220:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695887.225222:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298447575 00000100:00000001:1.0:1787695887.225223:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298458936 00000100:00000001:1.0:1787695887.225225:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298447575 00000100:00000001:1.0:1787695887.225226:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298458936 00000100:00000001:1.0:1787695887.225228:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695887.225229:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695887.225230:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695887.225232:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695887/real 1787695887] req@ffff8b14466f6a00 x1874531139539968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695887.225237:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695887.225238:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695887.225239:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695887.225240:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695887.225241:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695887.225242:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695887.225244:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6a00 x1874531139539968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695887.225249:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6a00 x1874531139539968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.225256:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695887.225257:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.225258:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695887.225259:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695887.225262:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6a00 x1874531139539968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.225266:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695887.225269:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6a00 x1874531139539968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.225273:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695887.225275:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.225276:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695887.225276:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.225277:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695887.225278:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695887.225280:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695887.225282:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695887.225283:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695887.225285:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6a00 x1874531139539968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695887.225290:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139539968:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695887.225292:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695887.225294:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5c00 x1874531139540096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.225298:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695887.225298:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695887.225302:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695887.225304:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695887.225305:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695887.225307:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139540096:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695887.225309:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695887.225310:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695887.225311:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695887.225313:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695887.225314:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695887.225316:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695887.225318:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695887.225320:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695887.225322:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695887.225324:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139540096, portal 10 00000100:00000001:1.0:1787695887.225325:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695887.225326:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150912 : -128555779400704 : ffff8b14466f5c00) 00000100:00000040:1.0:1787695887.225330:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5c00 x1874531139540096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.225334:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695887.225335:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695887.225337:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695887.225338:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139540096, offset 0 00000400:00000200:1.0:1787695887.225341:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695887.225343:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695887.225345:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695887.225347:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695887.225348:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695887.225350:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695887.225352:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695887.225354:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17029:17029:256:4294967295] 192.168.201.140@tcp LPNI seq info [17028:17028:8:4294967295] 00000400:00000200:1.0:1787695887.225356:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695887.225360:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695887.225500:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695887.225503:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695887.225506:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695887.225508:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1787695887.225510:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695887.225512:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000800:00000100:1.0:1787695887.225515:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695887.225516:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695887.225518:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695887.225520:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695887.225523:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695887.225524:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695887.225526:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695887.225529:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5c00 x1874531139540096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695912 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.225535:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695887.225537:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5c00 x1874531139540096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695912 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.225541:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695887.225542:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695887.225544:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695887.225546:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695887.225549:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695887.225550:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298447554 00000100:00000001:1.0:1787695887.225552:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298458936 00000100:00000001:1.0:1787695887.225553:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298447554 00000100:00000001:1.0:1787695887.225555:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298458936 00000100:00000001:1.0:1787695887.225556:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695887.225558:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695887.225559:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695887.225562:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695887/real 1787695887] req@ffff8b14466f5c00 x1874531139540096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695887.225576:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695887.225578:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695887.225580:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695887.225583:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695887.225584:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695887.225587:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695887.225592:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5c00 x1874531139540096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695887.225601:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5c00 x1874531139540096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.225608:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695887.225609:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.225611:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695887.225612:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695887.225614:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5c00 x1874531139540096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.225619:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695887.225622:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5c00 x1874531139540096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.225629:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695887.225632:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.225634:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695887.225635:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.225637:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695887.225638:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695887.225641:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695887.225645:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695887.225647:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695887.225658:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5c00 x1874531139540096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695887.225666:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139540096:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695887.225670:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.225672:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695887.225674:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6a00 x1874531139539968/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.225678:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695887.225679:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695887.225681:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695887.225683:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695887.225684:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695887.225685:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695887.225686:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695887.225687:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:1.0:1787695887.225690:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695887.225691:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695887.225692:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6a00. 00000100:00000001:1.0:1787695887.225693:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695887.225694:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.225695:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695887.225697:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5c00 x1874531139540096/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695912 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695887.225701:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695887.225702:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695887.225703:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695887.225704:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695887.225705:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695887.225705:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695887.225707:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695887.225708:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675400. 02000000:00000001:1.0:1787695887.225710:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695887.225714:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695887.225715:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5c00. 00000100:00000001:1.0:1787695887.225716:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695887.225717:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.225718:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695887.225722:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695887.225723:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695887.225724:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695887.225725:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695887.286442:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695887.286448:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695887.286449:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695887.286454:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695887.286457:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695887.286460:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695887.286462:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695887.286482:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695887.286484:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695887.926480:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695887.926487:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695887.926491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695887.926492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695887.926495:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695887.926496:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695887.926499:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695887.926500:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695887.926502:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695887.926503:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695887.926505:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695887.926506:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695887.926561:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695887.926565:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695887.926567:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695888.310481:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695888.310487:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695888.310488:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695888.310492:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695888.310495:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695888.310498:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695888.310500:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695888.310502:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695888.310504:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695888.950474:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695888.950480:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695888.950484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695888.950485:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695888.950488:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695888.950489:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695888.950491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695888.950492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695888.950494:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695888.950495:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695888.950497:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695888.950498:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695888.950550:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695888.950567:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695888.950570:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695889.334409:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695889.334415:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695889.334417:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695889.334421:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695889.334424:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695889.334427:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695889.334429:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695889.334431:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695889.334434:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695889.974491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695889.974496:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695889.974500:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695889.974501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695889.974504:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695889.974521:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695889.974524:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695889.974524:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695889.974527:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695889.974528:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695889.974530:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695889.974531:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695889.974593:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695889.974597:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695889.974600:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695890.358470:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695890.358476:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695890.358478:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695890.358482:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695890.358485:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695890.358488:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695890.358490:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695890.358492:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695890.358494:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695890.998459:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695890.998464:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695890.998468:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695890.998469:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695890.998472:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695890.998473:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695890.998475:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695890.998476:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695890.998479:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695890.998480:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695890.998483:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695890.998484:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695890.998533:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695890.998536:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695890.998538:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695891.382489:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695891.382497:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695891.382499:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695891.382522:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695891.382526:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695891.382529:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695891.382532:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695891.382536:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695891.382538:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695892.022467:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695892.022473:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695892.022478:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695892.022479:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695892.022481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695892.022482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695892.022485:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695892.022485:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695892.022488:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695892.022491:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695892.022493:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695892.022498:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695892.022500:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695892.022501:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695892.022503:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695892.022506:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695892.022507:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695892.022509:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695892.022511:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695892.022529:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695892.022614:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695892.022617:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695892.022620:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787695892.342434:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695892.342446:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787695892.342449:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695892.342452:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695892.342457:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695892.342464:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3780 00000400:00000200:3.0:1787695892.342473:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695892.342480:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695892.342483:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695892.342487:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695892.342491:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695892.342494:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695892.342497:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695892.342501:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695892.342503:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:3.0:1787695892.342506:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695892.342510:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787695892.342513:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787695892.342515:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787695892.342518:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695892.342521:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695892.342523:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695892.342525:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695892.342528:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787695892.342531:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695892.342550:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695892.342552:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695892.342556:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695892.342559:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695892.342561:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787695892.342564:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695892.342570:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6680. 00000020:00000040:3.0:1787695892.342574:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787695892.342578:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695892.342581:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695892.342584:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695892.342586:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695892.342591:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695892.342598:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675800. 00000100:00000001:3.0:1787695892.342605:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695892.342615:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6680 x1874531139540224/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695892.342627:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695892.342631:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695892.342636:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f6d2abf9] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6680 x1874531139540224/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695892.342658:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695892.342662:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695892.342670:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695892.342672:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695892.342674:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695892.342678:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695892.342681:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3780 00000400:00000200:3.0:1787695892.342686:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695892.342690:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:3.0:1787695892.342691:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695892.342696:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1787695892.342697:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695892.342699:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1787695892.342704:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695892.342707:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6680 x1874531139540224/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695892.342716:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695892.342720:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695892.342724:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695892.342724:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695892.342731:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787695892.342731:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:1.0:1787695892.342739:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1787695892.342741:0:38:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1787695892.342743:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695892.342745:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000100:00100000:1.0:1787695892.342749:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139540224:192.168.201.140@tcp:250:kworker.0 00000020:00000001:3.0:1787695892.342750:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695892.342755:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1787695892.342755:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 02000000:00000001:1.0:1787695892.342761:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:3.0:1787695892.342761:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 02000000:00000001:1.0:1787695892.342764:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695892.342766:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 02000000:00000001:1.0:1787695892.342767:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695892.342773:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 00000100:00000040:3.0:1787695892.342774:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695892.342776:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695892.342781:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695892.342785:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695892.342788:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000010:1.0:1787695892.342790:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695892.342795:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695892.342801:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139540224, portal 25 00000100:00000001:3.0:1787695892.342803:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695892.342805:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695892.342808:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930153600 : -128555779398016 : ffff8b14466f6680) 00000020:00000040:3.0:1787695892.342809:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:3.0:1787695892.342816:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000040:1.0:1787695892.342819:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6680 x1874531139540224/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695892.342820:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695892.342825:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695892.342831:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695892.342833:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787695892.342836:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695892.342842:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695892.342842:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:1.0:1787695892.342849:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000008:00100000:3.0:1787695892.342849:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695892.342856:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695892.342857:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139540224, offset 0 00000100:00000001:3.0:1787695892.342859:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695892.342865:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5880. 00000400:00000200:1.0:1787695892.342866:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000020:00000040:3.0:1787695892.342871:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695892.342876:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1787695892.342879:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787695892.342879:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695892.342884:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1787695892.342887:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 02000000:00000001:3.0:1787695892.342887:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695892.342891:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695892.342897:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675400. 00000400:00000200:1.0:1787695892.342898:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695892.342905:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:3.0:1787695892.342905:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695892.342908:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695892.342910:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695892.342915:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17030:17030:256:4294967295] 192.168.201.140@tcp LPNI seq info [17029:17029:8:4294967295] 00000400:00000200:1.0:1787695892.342920:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00100000:3.0:1787695892.342924:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5880 x1874531139540352/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695892.342936:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:3.0:1787695892.342945:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:1.0:1787695892.342949:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695892.342950:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1787695892.342954:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000100:00000040:3.0:1787695892.342959:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000827e7dbe] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5880 x1874531139540352/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1787695892.342965:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695892.342972:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000100:00000001:3.0:1787695892.342977:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1787695892.342981:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00080000:3.0:1787695892.342983:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1787695892.342985:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000100:00080000:3.0:1787695892.342990:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:1.0:1787695892.342995:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695892.342995:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000010:1.0:1787695892.343000:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000100:00080000:3.0:1787695892.343001:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1787695892.343007:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1787695892.343007:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695892.343013:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3780 00000400:00000200:1.0:1787695892.343014:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695892.343021:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695892.343024:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000400:00000200:3.0:1787695892.343025:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695892.343028:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:3.0:1787695892.343033:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000200:1.0:1787695892.343035:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6680 x1874531139540224/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695892.343038:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695892.343045:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695892.343048:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6680 x1874531139540224/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695917 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.343057:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695892.343060:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1787695892.343060:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695892.343065:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695892.343066:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695892.343071:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695892.343075:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695892.343077:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695892.343078:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298464056 00000020:00000001:3.0:1787695892.343083:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695892.343086:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000100:00000001:1.0:1787695892.343087:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298458936 00000020:00000001:3.0:1787695892.343089:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695892.343092:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298464056 00000020:00000040:3.0:1787695892.343093:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:3.0:1787695892.343097:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:3.0:1787695892.343101:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000001:1.0:1787695892.343104:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298458936 00000100:00000040:3.0:1787695892.343108:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695892.343109:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695892.343114:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695892.343115:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695892.343119:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:3.0:1787695892.343122:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00100000:1.0:1787695892.343126:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695892/real 1787695892] req@ffff8b14466f6680 x1874531139540224/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695892.343136:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000001:1.0:1787695892.343138:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695892.343140:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000040:3.0:1787695892.343143:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000400:00000010:1.0:1787695892.343144:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695892.343148:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695892.343152:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00080000:3.0:1787695892.343153:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695892.343155:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787695892.343160:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695892.343162:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6680 x1874531139540224/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695892.343165:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695892.343170:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1787695892.343173:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6680 x1874531139540224/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695892.343175:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695892.343181:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695892.343185:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000008:00100000:3.0:1787695892.343185:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787695892.343188:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1787695892.343191:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695892.343192:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695892.343194:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695892.343196:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695892.343199:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6680 x1874531139540224/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1787695892.343203:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4a80. 00000100:00000001:1.0:1787695892.343205:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1787695892.343209:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000100:00000040:1.0:1787695892.343211:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6680 x1874531139540224/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695892.343216:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695892.343219:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695892.343220:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787695892.343224:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787695892.343225:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.343229:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1787695892.343230:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787695892.343231:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.343234:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:3.0:1787695892.343235:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695892.343241:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000010:3.0:1787695892.343242:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 00000100:00080000:1.0:1787695892.343244:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787695892.343248:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695892.343252:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695892.343258:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787695892.343259:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4a80 x1874531139540480/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1787695892.343269:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6680 x1874531139540224/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695892.343272:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695892.343277:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695892.343297:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139540224:192.168.201.140@tcp:250:kworker.0 00000100:00000001:1.0:1787695892.343305:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.343309:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695892.343313:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ad0d67] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4a80 x1874531139540480/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695892.343316:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6680 x1874531139540224/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.343341:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1787695892.343344:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695892.343346:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:3.0:1787695892.343348:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000010:1.0:1787695892.343351:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 00000100:00080000:3.0:1787695892.343353:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787695892.343357:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1787695892.343359:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:3.0:1787695892.343359:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:1.0:1787695892.343387:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695892.343392:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:1.0:1787695892.343394:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695892.343396:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675800. 02000000:00000001:1.0:1787695892.343400:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695892.343402:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695892.343405:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6680. 00000100:00000001:3.0:1787695892.343407:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695892.343409:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695892.343412:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695892.343414:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3780 00000100:00000001:1.0:1787695892.343415:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787695892.343423:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695892.343425:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695892.343428:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1787695892.343429:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695892.343431:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787695892.343433:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695892.343435:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695892.343437:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1787695892.343441:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695892.343445:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5880 x1874531139540352/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695892.343446:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695892.343452:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695892.343457:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695892.343460:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695892.343463:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:3.0:1787695892.343467:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695892.343470:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000100:00000001:1.0:1787695892.343474:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1787695892.343475:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:1.0:1787695892.343476:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695892.343479:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 02000000:00000001:1.0:1787695892.343480:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695892.343484:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1787695892.343485:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695892.343488:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695892.343489:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1787695892.343493:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139540352:192.168.201.140@tcp:38:kworker.0 00000100:00000040:3.0:1787695892.343496:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695892.343499:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787695892.343500:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:1.0:1787695892.343502:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695892.343506:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695892.343506:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 02000000:00000001:1.0:1787695892.343511:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1787695892.343513:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000010:1.0:1787695892.343515:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695892.343519:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695892.343520:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695892.343524:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1787695892.343525:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695892.343529:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 02000000:00000001:3.0:1787695892.343529:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1787695892.343532:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 02000000:00000001:3.0:1787695892.343533:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695892.343536:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139540352, portal 10 00000100:00000001:3.0:1787695892.343538:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695892.343541:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695892.343543:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150016 : -128555779401600 : ffff8b14466f5880) 00000008:00100000:3.0:1787695892.343543:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787695892.343550:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695892.343553:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695892.343554:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5880 x1874531139540352/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1787695892.343560:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4000. 00000100:00000001:1.0:1787695892.343563:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695892.343566:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695892.343570:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000020:00000040:3.0:1787695892.343573:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000100:00000200:1.0:1787695892.343575:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139540352, offset 0 02000000:00000001:3.0:1787695892.343578:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695892.343581:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695892.343584:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695892.343585:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695892.343590:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695892.343593:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695892.343594:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000010:3.0:1787695892.343600:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676000. 00000400:00000200:1.0:1787695892.343601:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:3.0:1787695892.343607:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695892.343608:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695892.343614:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00100000:3.0:1787695892.343618:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4000 x1874531139540608/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695892.343619:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695892.343624:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:3.0:1787695892.343631:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1787695892.343635:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17031:17031:256:4294967295] 192.168.201.140@tcp LPNI seq info [17030:17030:8:4294967295] 00000100:00000001:3.0:1787695892.343636:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695892.343645:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695892.343645:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5a8f356] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4000 x1874531139540608/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695892.343659:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:3.0:1787695892.343669:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695892.343670:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00080000:3.0:1787695892.343675:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:1.0:1787695892.343676:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695892.343683:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00080000:3.0:1787695892.343683:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1787695892.343686:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:1.0:1787695892.343688:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000100:00080000:3.0:1787695892.343689:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.1:1787695892.343695:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787695892.343696:0:38:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:1.1:1787695892.343699:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000100:00080000:3.0:1787695892.343702:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3780 00000800:00000100:1.0:1787695892.343707:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695892.343710:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:3.0:1787695892.343714:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695892.343715:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695892.343722:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1787695892.343722:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695892.343728:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1787695892.343729:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695892.343732:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000040:3.0:1787695892.343734:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695892.343736:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787695892.343739:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1787695892.343743:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5880 x1874531139540352/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695892.343745:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695892.343750:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695892.343756:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695892.343758:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000040:3.0:1787695892.343759:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000100:00000040:1.0:1787695892.343763:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5880 x1874531139540352/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695917 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.343771:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695892.343771:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695892.343775:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:3.0:1787695892.343777:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000100:00000001:1.0:1787695892.343778:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695892.343784:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:1.0:1787695892.343785:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000020:00000001:3.0:1787695892.343790:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000400:00000200:1.0:1787695892.343791:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695892.343795:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298453958 00000100:00000040:3.0:1787695892.343799:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695892.343800:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298464056 00000100:00000001:1.0:1787695892.343805:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298453958 00000100:00000001:3.0:1787695892.343806:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695892.343809:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298464056 00000100:00000040:3.0:1787695892.343812:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695892.343815:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695892.343816:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695892.343818:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:3.0:1787695892.343821:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695892.343828:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695892.343828:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695892.343832:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695892.343834:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1787695892.343837:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695892.343841:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695892.343844:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695892/real 1787695892] req@ffff8b14466f5880 x1874531139540352/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695892.343844:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695892.343848:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787695892.343854:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695892.343859:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695892.343866:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4700. 00000400:00000001:1.0:1787695892.343867:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695892.343870:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000040:3.0:1787695892.343872:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 00000400:00000010:1.0:1787695892.343873:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695892.343878:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 02000000:00000001:3.0:1787695892.343878:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695892.343881:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695892.343882:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:3.0:1787695892.343887:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787695892.343888:0:2402:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1787695892.343892:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:1.0:1787695892.343894:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5880 x1874531139540352/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695892.343895:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695892.343902:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 00000100:00000200:1.0:1787695892.343907:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5880 x1874531139540352/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695892.343908:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695892.343916:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695892.343919:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787695892.343919:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4700 x1874531139540736/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695892.343922:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695892.343924:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695892.343931:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5880 x1874531139540352/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695892.343941:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695892.343944:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695892.343946:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695892.343951:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5880 x1874531139540352/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695892.343955:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008818c2cc] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4700 x1874531139540736/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.343958:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695892.343960:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.343961:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695892.343962:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.343966:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695892.343968:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695892.343973:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787695892.343978:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695892.343984:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695892.343987:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00080000:3.0:1787695892.343990:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787695892.343991:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787695892.343994:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695892.343998:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1787695892.343999:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5880 x1874531139540352/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695892.344005:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695892.344011:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3780 00000100:00100000:1.0:1787695892.344013:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139540352:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695892.344019:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000400:00000200:3.0:1787695892.344019:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:1.0:1787695892.344026:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4a80 x1874531139540480/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695892.344027:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695892.344033:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695892.344039:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1787695892.344041:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695892.344042:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695892.344047:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787695892.344048:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695892.344051:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1787695892.344053:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695892.344054:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695892.344058:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00100000:1.0:1787695892.344061:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139540480:192.168.201.140@tcp:38:kworker.0 00000020:00000001:3.0:1787695892.344062:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695892.344065:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000100:00000001:1.0:1787695892.344066:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:3.0:1787695892.344069:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1787695892.344070:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:3.0:1787695892.344072:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 02000000:00000001:1.0:1787695892.344073:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695892.344077:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1787695892.344078:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 02000000:00000010:1.0:1787695892.344081:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 00000020:00000001:3.0:1787695892.344083:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 02000000:00000001:1.0:1787695892.344084:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695892.344090:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695892.344091:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695892.344095:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000100:00000001:3.0:1787695892.344096:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1787695892.344099:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000040:3.0:1787695892.344100:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000200:1.0:1787695892.344104:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139540480, portal 10 00000100:00000001:3.0:1787695892.344104:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695892.344109:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000040:3.0:1787695892.344111:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00000001:1.0:1787695892.344112:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930146432 : -128555779405184 : ffff8b14466f4a80) 00000100:00080000:3.0:1787695892.344117:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000040:1.0:1787695892.344121:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4a80 x1874531139540480/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695892.344122:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695892.344127:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787695892.344132:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1787695892.344138:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695892.344138:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1787695892.344140:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 02000000:00000001:3.0:1787695892.344142:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695892.344144:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139540480, offset 0 00000100:00000001:3.0:1787695892.344146:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:1.0:1787695892.344150:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000008:00100000:3.0:1787695892.344151:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695892.344155:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695892.344156:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695892.344160:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695892.344163:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695892.344167:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000010:3.0:1787695892.344167:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5500. 00000400:00000200:1.0:1787695892.344173:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000020:00000040:3.0:1787695892.344173:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000400:00000200:1.0:1787695892.344178:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 02000000:00000001:3.0:1787695892.344179:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695892.344182:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695892.344186:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:3.0:1787695892.344186:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695892.344190:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695892.344194:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695892.344195:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17032:17032:256:4294967295] 192.168.201.140@tcp LPNI seq info [17031:17031:8:4294967295] 02000000:00000010:3.0:1787695892.344199:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 00000400:00000200:1.0:1787695892.344204:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695892.344205:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695892.344215:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5500 x1874531139540864/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695892.344219:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:3.0:1787695892.344234:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:1.0:1787695892.344235:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695892.344239:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1787695892.344243:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000100:00000040:3.0:1787695892.344246:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a777d68] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5500 x1874531139540864/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1787695892.344254:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695892.344261:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000100:00000001:3.0:1787695892.344266:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695892.344272:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1787695892.344272:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695892.344291:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000100:00080000:3.0:1787695892.344298:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695892.344301:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695892.344305:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787695892.344309:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695892.344311:0:38:0:(import.c:485:import_select_connection()) Process entered 00000800:00000010:1.0:1787695892.344313:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000100:00080000:3.0:1787695892.344317:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3780 00000400:00000200:1.0:1787695892.344318:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695892.344327:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1787695892.344329:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695892.344336:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000200:1.0:1787695892.344337:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000100:00080000:3.0:1787695892.344339:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000400:00000010:1.0:1787695892.344340:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695892.344343:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695892.344347:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4a80 x1874531139540480/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695892.344355:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695892.344358:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787695892.344360:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695892.344377:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4a80 x1874531139540480/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695917 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.344383:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695892.344385:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1787695892.344389:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695892.344393:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695892.344395:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695892.344398:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000040:1.0:1787695892.344399:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000020:00000040:3.0:1787695892.344401:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000020:00000001:3.0:1787695892.344405:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:1.0:1787695892.344406:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695892.344410:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298452807 00000020:00000040:3.0:1787695892.344411:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000100:00000001:1.0:1787695892.344413:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298464056 00000100:00000001:1.0:1787695892.344417:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298452807 00000020:00000001:3.0:1787695892.344417:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:1.0:1787695892.344423:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298464056 00000020:00000001:3.0:1787695892.344424:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:1.0:1787695892.344425:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695892.344429:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695892.344431:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695892.344432:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787695892.344437:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1787695892.344438:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695892/real 1787695892] req@ffff8b14466f4a80 x1874531139540480/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695892.344443:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000001:1.0:1787695892.344451:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787695892.344451:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000001:1.0:1787695892.344454:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695892.344457:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000020:00000040:3.0:1787695892.344457:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000400:00000200:1.0:1787695892.344463:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000100:00080000:3.0:1787695892.344465:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000010:1.0:1787695892.344466:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695892.344470:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787695892.344472:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695892.344475:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4a80 x1874531139540480/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695892.344477:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695892.344483:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1787695892.344486:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4a80 x1874531139540480/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695892.344488:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695892.344493:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695892.344496:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787695892.344497:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695892.344499:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.344503:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000001:3.0:1787695892.344503:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695892.344506:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695892.344509:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695892.344510:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4a80 x1874531139540480/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1787695892.344517:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5f80. 00000100:00000001:1.0:1787695892.344518:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695892.344525:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4a80 x1874531139540480/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1787695892.344525:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:3.0:1787695892.344530:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695892.344534:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695892.344538:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787695892.344540:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1787695892.344542:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787695892.344544:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787695892.344546:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695892.344547:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695892.344549:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.344552:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695892.344553:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695892.344555:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 02000000:00000010:3.0:1787695892.344555:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 00000100:00080000:1.0:1787695892.344558:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695892.344560:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695892.344565:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4a80 x1874531139540480/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695892.344568:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695892.344580:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5f80 x1874531139540992/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787695892.344583:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139540480:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695892.344595:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695892.344599:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695892.344600:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787695892.344604:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695892.344609:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5880 x1874531139540352/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695892.344614:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006e7bde76] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5f80 x1874531139540992/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.344617:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695892.344620:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695892.344623:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695892.344627:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695892.344630:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695892.344633:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000100:00000001:3.0:1787695892.344633:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787695892.344636:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695892.344639:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675400. 00000100:00000040:3.0:1787695892.344639:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1787695892.344644:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695892.344647:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695892.344648:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5880. 00000100:00000040:3.0:1787695892.344648:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1787695892.344653:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695892.344655:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695892.344655:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787695892.344660:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1787695892.344665:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695892.344669:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3790) 00000100:00000040:1.0:1787695892.344670:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4a80 x1874531139540480/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.344678:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695892.344679:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695892.344681:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695892.344682:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695892.344683:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695892.344684:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695892.344685:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695892.344686:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:1.0:1787695892.344689:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695892.344690:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695892.344690:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4a80. 00000100:00000001:1.0:1787695892.344692:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695892.344693:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.344694:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.344696:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695892.344697:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695892.344699:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695892.344700:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695892.344701:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695892.344703:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4000 x1874531139540608/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.344708:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695892.344709:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695892.344710:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695892.344712:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695892.344713:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695892.344715:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139540608:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695892.344717:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695892.344718:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695892.344720:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695892.344721:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695892.344723:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695892.344727:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695892.344730:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695892.344732:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695892.344734:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695892.344737:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139540608, portal 4 00000100:00000001:1.0:1787695892.344738:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695892.344739:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930143744 : -128555779407872 : ffff8b14466f4000) 00000100:00000040:1.0:1787695892.344743:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4000 x1874531139540608/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.344748:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695892.344749:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695892.344751:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695892.344752:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139540608, offset 0 00000400:00000200:1.0:1787695892.344755:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695892.344758:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695892.344760:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695892.344762:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695892.344764:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695892.344765:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695892.344767:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695892.344769:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17033:17033:256:4294967295] 192.168.201.140@tcp LPNI seq info [17032:17032:8:4294967295] 00000400:00000200:1.0:1787695892.344772:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695892.344776:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695892.344779:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695892.344781:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695892.344784:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695892.344786:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1787695892.344789:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695892.344791:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1787695892.344793:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695892.344795:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695892.344797:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695892.344799:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695892.344804:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695892.344805:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695892.344807:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695892.344810:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4000 x1874531139540608/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.344818:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695892.344821:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4000 x1874531139540608/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695917 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.344828:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695892.344830:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695892.344834:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695892.344838:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695892.344841:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695892.344843:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298452837 00000100:00000001:1.0:1787695892.344846:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298464056 00000100:00000001:1.0:1787695892.344849:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298452837 00000100:00000001:1.0:1787695892.344851:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298464056 00000100:00000001:1.0:1787695892.344853:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695892.344856:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695892.344857:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695892.344860:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695892/real 1787695892] req@ffff8b14466f4000 x1874531139540608/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695892.344866:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695892.344867:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695892.344869:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695892.344870:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695892.344871:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695892.344873:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695892.344875:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4000 x1874531139540608/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695892.344889:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4000 x1874531139540608/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.344894:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695892.344895:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.344896:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695892.344897:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695892.344900:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4000 x1874531139540608/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.344905:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695892.344907:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4000 x1874531139540608/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.344911:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695892.344913:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.344913:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695892.344914:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.344915:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695892.344916:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695892.344918:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695892.344920:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695892.344921:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695892.344923:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4000 x1874531139540608/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695892.344928:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139540608:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695892.344931:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695892.344933:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4700 x1874531139540736/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.344945:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695892.344946:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695892.344948:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695892.344949:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695892.344950:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695892.344952:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139540736:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695892.344954:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695892.344955:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695892.344957:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695892.344958:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695892.344960:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695892.344961:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695892.344963:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695892.344965:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695892.344967:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695892.344969:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139540736, portal 4 00000100:00000001:1.0:1787695892.344971:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695892.344972:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930145536 : -128555779406080 : ffff8b14466f4700) 00000100:00000040:1.0:1787695892.344976:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4700 x1874531139540736/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.344980:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695892.344981:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695892.344983:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695892.344984:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139540736, offset 0 00000400:00000200:1.0:1787695892.344987:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695892.344989:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695892.344991:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695892.344994:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695892.344995:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695892.344997:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695892.344999:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695892.345004:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17034:17034:256:4294967295] 192.168.201.140@tcp LPNI seq info [17033:17033:8:4294967295] 00000400:00000200:1.0:1787695892.345007:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695892.345012:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695892.345015:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695892.345017:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695892.345020:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695892.345022:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1787695892.345024:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695892.345026:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1787695892.345029:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695892.345030:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695892.345032:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695892.345034:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695892.345036:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695892.345037:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695892.345039:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695892.345041:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4700 x1874531139540736/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345047:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695892.345049:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4700 x1874531139540736/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695917 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345053:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695892.345054:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695892.345056:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695892.345058:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695892.345059:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695892.345061:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298452509 00000100:00000001:1.0:1787695892.345062:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298464056 00000100:00000001:1.0:1787695892.345064:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298452509 00000100:00000001:1.0:1787695892.345065:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298464056 00000100:00000001:1.0:1787695892.345067:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695892.345071:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695892.345072:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695892.345075:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695892/real 1787695892] req@ffff8b14466f4700 x1874531139540736/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695892.345080:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695892.345081:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695892.345082:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695892.345084:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695892.345085:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695892.345086:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695892.345088:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4700 x1874531139540736/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695892.345094:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4700 x1874531139540736/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345097:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695892.345098:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.345099:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695892.345100:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695892.345102:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4700 x1874531139540736/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345106:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695892.345108:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4700 x1874531139540736/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345112:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695892.345114:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.345115:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695892.345116:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.345118:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695892.345119:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695892.345120:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695892.345125:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695892.345127:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695892.345130:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4700 x1874531139540736/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695892.345135:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139540736:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695892.345137:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695892.345139:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5500 x1874531139540864/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345143:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695892.345143:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695892.345145:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695892.345146:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695892.345147:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695892.345149:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139540864:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695892.345151:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695892.345152:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695892.345153:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695892.345154:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695892.345156:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544c00. 02000000:00000001:1.0:1787695892.345157:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695892.345159:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695892.345161:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695892.345163:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695892.345165:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139540864, portal 10 00000100:00000001:1.0:1787695892.345166:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695892.345167:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930149120 : -128555779402496 : ffff8b14466f5500) 00000100:00000040:1.0:1787695892.345171:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5500 x1874531139540864/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345175:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695892.345179:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695892.345181:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695892.345183:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139540864, offset 0 00000400:00000200:1.0:1787695892.345185:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695892.345188:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695892.345190:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695892.345192:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695892.345194:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695892.345195:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695892.345197:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695892.345199:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17035:17035:256:4294967295] 192.168.201.140@tcp LPNI seq info [17034:17034:8:4294967295] 00000400:00000200:1.0:1787695892.345201:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695892.345205:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695892.345208:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695892.345210:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695892.345212:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695892.345214:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1787695892.345217:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695892.345218:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1787695892.345221:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695892.345222:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695892.345224:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695892.345226:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695892.345228:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695892.345229:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695892.345230:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695892.345233:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5500 x1874531139540864/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345238:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695892.345240:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5500 x1874531139540864/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695917 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345245:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695892.345246:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695892.345251:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695892.345253:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695892.345256:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695892.345257:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298452695 00000100:00000001:1.0:1787695892.345259:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298464056 00000100:00000001:1.0:1787695892.345260:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298452695 00000100:00000001:1.0:1787695892.345262:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298464056 00000100:00000001:1.0:1787695892.345263:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695892.345265:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695892.345266:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695892.345268:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695892/real 1787695892] req@ffff8b14466f5500 x1874531139540864/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695892.345273:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695892.345273:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695892.345275:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695892.345276:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695892.345277:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695892.345293:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695892.345297:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5500 x1874531139540864/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695892.345303:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5500 x1874531139540864/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345307:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695892.345308:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.345309:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695892.345310:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695892.345312:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5500 x1874531139540864/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345316:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695892.345319:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5500 x1874531139540864/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345330:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695892.345333:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.345335:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695892.345336:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.345338:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695892.345340:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695892.345342:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695892.345347:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695892.345349:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695892.345353:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5500 x1874531139540864/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695892.345361:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139540864:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695892.345381:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695892.345386:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5f80 x1874531139540992/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345394:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695892.345396:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695892.345399:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695892.345401:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695892.345404:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695892.345407:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139540992:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695892.345411:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695892.345414:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695892.345416:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695892.345418:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695892.345423:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545000. 02000000:00000001:1.0:1787695892.345425:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695892.345436:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695892.345438:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695892.345441:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695892.345443:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139540992, portal 10 00000100:00000001:1.0:1787695892.345445:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695892.345445:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930151808 : -128555779399808 : ffff8b14466f5f80) 00000100:00000040:1.0:1787695892.345450:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5f80 x1874531139540992/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345454:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695892.345456:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695892.345457:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695892.345459:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139540992, offset 0 00000400:00000200:1.0:1787695892.345461:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695892.345464:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695892.345466:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695892.345468:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695892.345470:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695892.345471:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695892.345473:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695892.345475:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17036:17036:256:4294967295] 192.168.201.140@tcp LPNI seq info [17035:17035:8:4294967295] 00000400:00000200:1.0:1787695892.345478:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695892.345482:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695892.345485:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695892.345487:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695892.345489:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695892.345491:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1787695892.345494:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695892.345496:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1787695892.345499:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695892.345500:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695892.345502:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695892.345504:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695892.345506:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695892.345511:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695892.345513:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695892.345516:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5f80 x1874531139540992/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695917 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345521:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695892.345523:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5f80 x1874531139540992/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695917 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345528:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695892.345529:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695892.345531:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695892.345533:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695892.345535:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695892.345537:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298452674 00000100:00000001:1.0:1787695892.345538:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298464056 00000100:00000001:1.0:1787695892.345540:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298452674 00000100:00000001:1.0:1787695892.345541:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298464056 00000100:00000001:1.0:1787695892.345542:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695892.345544:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695892.345545:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695892.345547:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695892/real 1787695892] req@ffff8b14466f5f80 x1874531139540992/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695892.345552:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695892.345553:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695892.345554:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695892.345556:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695892.345557:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695892.345558:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695892.345561:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5f80 x1874531139540992/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695892.345570:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5f80 x1874531139540992/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345581:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695892.345583:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.345585:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695892.345587:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695892.345591:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5f80 x1874531139540992/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345599:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695892.345603:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5f80 x1874531139540992/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345609:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695892.345611:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.345612:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695892.345613:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.345614:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695892.345616:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695892.345617:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695892.345620:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695892.345622:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695892.345624:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5f80 x1874531139540992/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695892.345629:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139540992:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695892.345632:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.345633:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695892.345635:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4000 x1874531139540608/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345639:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695892.345644:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695892.345645:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695892.345647:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695892.345649:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695892.345650:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695892.345651:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695892.345653:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676000. 02000000:00000001:1.0:1787695892.345655:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695892.345656:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695892.345658:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4000. 00000100:00000001:1.0:1787695892.345659:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695892.345660:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.345661:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695892.345664:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4700 x1874531139540736/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345668:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695892.345669:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695892.345670:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695892.345671:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695892.345672:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695892.345673:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695892.345674:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695892.345675:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:1.0:1787695892.345676:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695892.345677:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695892.345678:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4700. 00000100:00000001:1.0:1787695892.345679:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695892.345680:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.345680:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695892.345682:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5500 x1874531139540864/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345686:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695892.345687:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695892.345688:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544c00. 02000000:00000001:1.0:1787695892.345689:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695892.345689:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695892.345693:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695892.345694:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695892.345695:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:1.0:1787695892.345697:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695892.345698:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695892.345699:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5500. 00000100:00000001:1.0:1787695892.345700:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695892.345701:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.345702:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695892.345704:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5f80 x1874531139540992/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695917 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695892.345707:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695892.345708:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695892.345709:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545000. 02000000:00000001:1.0:1787695892.345711:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695892.345712:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695892.345712:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695892.345713:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695892.345714:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:1.0:1787695892.345716:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695892.345716:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695892.345717:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5f80. 00000100:00000001:1.0:1787695892.345718:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695892.345719:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.345720:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695892.345723:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695892.345724:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695892.345725:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695892.345726:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695892.406476:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695892.406483:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695892.406485:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695892.406489:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695892.406492:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695892.406495:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695892.406497:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695892.406499:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695892.406501:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695893.046477:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695893.046484:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695893.046491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695893.046493:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695893.046497:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695893.046498:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695893.046502:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695893.046503:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695893.046527:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695893.046528:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695893.046532:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695893.046533:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695893.046593:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695893.046596:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695893.046598:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695893.430428:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695893.430435:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695893.430436:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695893.430440:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695893.430443:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695893.430446:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695893.430449:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695893.430451:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695893.430453:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695894.070472:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695894.070478:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695894.070480:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695894.070483:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695894.070487:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695894.070489:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695894.070491:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695894.070492:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695894.070495:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695894.070496:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695894.070497:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695894.070498:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695894.070500:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695894.070501:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695894.070502:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695894.070503:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695894.070506:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695894.070506:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695894.070509:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695894.070510:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695894.070512:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695894.070513:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695894.070515:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695894.070515:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695894.070579:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695894.070584:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695894.070588:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695894.454526:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695894.454537:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695894.454540:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695894.454548:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695894.454553:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695894.454558:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695894.454561:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695894.454565:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695894.454568:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695895.094466:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695895.094474:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695895.094479:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695895.094481:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695895.094484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695895.094485:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695895.094488:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695895.094489:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695895.094492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695895.094493:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695895.094517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695895.094518:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695895.094578:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695895.094582:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695895.094585:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695895.478511:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695895.478521:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695895.478523:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695895.478529:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695895.478532:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695895.478537:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695895.478539:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695895.478541:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695895.478544:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695896.118521:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695896.118529:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695896.118534:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695896.118535:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695896.118539:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695896.118540:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695896.118543:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695896.118544:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695896.118547:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695896.118548:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695896.118551:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695896.118552:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695896.118616:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695896.118622:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695896.118627:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695896.502530:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695896.502539:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695896.502541:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695896.502546:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695896.502550:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695896.502554:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695896.502556:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695896.502559:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695896.502562:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695897.142471:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695897.142479:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695897.142484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695897.142485:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695897.142488:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695897.142492:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695897.142494:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695897.142501:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695897.142503:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695897.142504:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695897.142508:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695897.142510:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695897.142512:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695897.142514:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695897.142517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695897.142518:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695897.142519:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695897.142520:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695897.142520:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695897.142521:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695897.142522:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695897.142523:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695897.142525:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695897.142526:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695897.142529:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695897.142550:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695897.142554:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695897.142555:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695897.142556:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695897.142558:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695897.142559:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695897.142560:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695897.142561:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695897.142562:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695897.142563:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695897.142564:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695897.142624:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695897.142628:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695897.142631:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787695897.462586:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695897.462604:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787695897.462608:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695897.462614:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695897.462649:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695897.462663:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3785 00000400:00000200:3.0:1787695897.462692:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695897.462705:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695897.462709:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695897.462713:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.462718:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695897.462723:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.462726:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695897.462732:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695897.462735:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:3.0:1787695897.462739:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695897.462743:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787695897.462747:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787695897.462750:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787695897.462754:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.462757:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695897.462758:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.462761:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695897.462765:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787695897.462768:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695897.462771:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695897.462775:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695897.462780:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695897.462784:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695897.462787:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787695897.462790:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695897.462798:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6d80. 00000020:00000040:3.0:1787695897.462803:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787695897.462808:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695897.462810:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695897.462821:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695897.462825:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695897.462832:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695897.462843:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674800. 00000100:00000001:3.0:1787695897.462855:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695897.462871:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6d80 x1874531139541120/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695897.462886:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695897.462893:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695897.462900:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c3be57b] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6d80 x1874531139541120/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695897.462920:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695897.462924:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695897.462927:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695897.462929:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695897.462931:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695897.462936:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695897.462939:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3785 00000400:00000200:3.0:1787695897.462943:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695897.462946:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695897.462950:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695897.462953:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.462956:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695897.462958:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.462960:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695897.462964:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695897.462965:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:3.0:1787695897.462967:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695897.462970:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:3.0:1787695897.462973:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:3.0:1787695897.462975:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:3.0:1787695897.462986:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.462989:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695897.462991:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.462994:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695897.462996:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:3.0:1787695897.463000:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463003:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695897.463005:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695897.463012:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695897.463014:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695897.463017:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695897.463031:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695897.463035:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695897.463037:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695897.463045:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5180. 00000020:00000040:3.0:1787695897.463048:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695897.463051:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695897.463053:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695897.463055:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695897.463057:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695897.463059:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695897.463067:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 00000100:00000001:3.0:1787695897.463071:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695897.463077:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5180 x1874531139541248/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695897.463085:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695897.463089:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695897.463093:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a115abf4] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5180 x1874531139541248/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695897.463102:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695897.463104:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695897.463115:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695897.463116:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695897.463118:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695897.463123:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695897.463128:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3785 00000400:00000200:3.0:1787695897.463133:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695897.463136:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695897.463138:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695897.463142:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463144:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695897.463145:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463148:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695897.463150:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695897.463151:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:3.0:1787695897.463153:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695897.463155:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:3.0:1787695897.463158:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:3.0:1787695897.463160:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:3.0:1787695897.463163:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463165:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695897.463168:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463170:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695897.463173:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:3.0:1787695897.463175:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463178:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695897.463180:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695897.463183:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695897.463186:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695897.463188:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695897.463203:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695897.463219:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695897.463221:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695897.463225:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4380. 00000020:00000040:3.0:1787695897.463227:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695897.463230:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695897.463232:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695897.463233:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695897.463235:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695897.463237:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695897.463243:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 00000100:00000001:3.0:1787695897.463246:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695897.463252:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4380 x1874531139541376/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695897.463260:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695897.463263:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695897.463267:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d2982c15] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4380 x1874531139541376/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695897.463275:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695897.463277:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695897.463281:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787695897.463283:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695897.463284:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695897.463288:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695897.463290:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3785 00000400:00000200:3.0:1787695897.463295:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695897.463298:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695897.463301:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695897.463303:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463306:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695897.463309:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463317:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695897.463319:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695897.463321:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:3.0:1787695897.463322:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695897.463325:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:3.0:1787695897.463327:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:3.0:1787695897.463329:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:3.0:1787695897.463333:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463335:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695897.463336:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463339:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695897.463342:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:3.0:1787695897.463345:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463348:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695897.463350:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695897.463353:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695897.463355:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695897.463358:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695897.463362:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787695897.463405:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695897.463407:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695897.463413:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5f80. 00000020:00000040:3.0:1787695897.463415:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:3.0:1787695897.463418:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695897.463420:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695897.463422:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695897.463424:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695897.463425:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695897.463431:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 00000100:00000001:3.0:1787695897.463434:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695897.463439:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5f80 x1874531139541504/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695897.463459:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695897.463461:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695897.463466:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006e7bde76] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5f80 x1874531139541504/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695897.463473:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695897.463476:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695897.463481:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1787695897.463482:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695897.463484:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695897.463488:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695897.463490:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3785 00000400:00000200:3.0:1787695897.463495:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695897.463499:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695897.463501:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695897.463505:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463508:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695897.463510:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463513:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695897.463515:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695897.463516:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:3.0:1787695897.463519:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695897.463522:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:3.0:1787695897.463525:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:3.0:1787695897.463527:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:3.0:1787695897.463531:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463533:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695897.463535:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463537:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695897.463545:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:3.0:1787695897.463549:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463552:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695897.463554:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695897.463558:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695897.463560:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695897.463563:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695897.463566:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787695897.463568:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695897.463571:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695897.463574:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5500. 00000020:00000040:3.0:1787695897.463577:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:3.0:1787695897.463580:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695897.463582:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695897.463583:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695897.463585:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695897.463587:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695897.463591:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676000. 00000100:00000001:3.0:1787695897.463595:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695897.463600:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5500 x1874531139541632/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695897.463607:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695897.463609:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695897.463613:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a777d68] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5500 x1874531139541632/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695897.463622:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695897.463625:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695897.463629:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695897.463631:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695897.463632:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695897.463635:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695897.463645:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3785 00000400:00000200:3.0:1787695897.463650:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695897.463653:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695897.463657:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695897.463660:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463662:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695897.463664:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463667:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695897.463669:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695897.463671:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:3.0:1787695897.463673:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695897.463676:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000020:00000001:3.0:1787695897.463678:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000020:00000001:3.0:1787695897.463680:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000040:3.0:1787695897.463684:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463686:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695897.463688:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463691:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695897.463693:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:3.0:1787695897.463696:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463698:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695897.463701:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695897.463704:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695897.463706:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695897.463709:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695897.463711:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695897.463714:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695897.463716:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695897.463720:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4700. 00000020:00000040:3.0:1787695897.463723:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:3.0:1787695897.463726:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695897.463733:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695897.463736:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695897.463738:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695897.463740:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695897.463744:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 00000100:00000001:3.0:1787695897.463748:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695897.463755:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4700 x1874531139541760/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695897.463762:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695897.463764:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695897.463768:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008818c2cc] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4700 x1874531139541760/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695897.463776:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695897.463778:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695897.463781:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695897.463783:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695897.463785:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695897.463788:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695897.463791:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3785 00000400:00000200:3.0:1787695897.463795:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695897.463798:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695897.463801:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695897.463804:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463807:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695897.463808:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463810:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695897.463812:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695897.463813:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000020:00000001:3.0:1787695897.463815:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695897.463818:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:3.0:1787695897.463819:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000020:00000001:3.0:1787695897.463827:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:3.0:1787695897.463832:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463834:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695897.463836:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463838:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695897.463841:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00080000:3.0:1787695897.463843:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695897.463846:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695897.463848:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695897.463852:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695897.463855:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695897.463857:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695897.463859:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695897.463862:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695897.463863:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695897.463867:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4000. 00000020:00000040:3.0:1787695897.463869:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:3.0:1787695897.463871:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695897.463873:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695897.463876:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695897.463877:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695897.463879:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695897.463883:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675400. 00000100:00000001:3.0:1787695897.463886:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695897.463890:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4000 x1874531139541888/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695897.463897:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695897.463899:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695897.463903:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5a8f356] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4000 x1874531139541888/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695897.463909:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695897.463918:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787695897.463922:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695897.463925:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787695897.463929:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695897.463935:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3795) 00000100:00000001:1.0:1787695897.464399:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695897.464404:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695897.464408:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695897.464417:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6d80 x1874531139541120/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.464426:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695897.464428:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695897.464432:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695897.464436:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695897.464438:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695897.464442:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139541120:192.168.201.140@tcp:250:kworker.0 00000100:00000001:1.0:1787695897.464447:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695897.464452:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695897.464455:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695897.464459:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695897.464468:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545000. 02000000:00000001:1.0:1787695897.464471:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695897.464479:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695897.464484:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695897.464492:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695897.464502:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139541120, portal 25 00000100:00000001:1.0:1787695897.464504:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695897.464506:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930155392 : -128555779396224 : ffff8b14466f6d80) 00000100:00000040:1.0:1787695897.464512:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6d80 x1874531139541120/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.464529:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695897.464532:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695897.464535:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695897.464539:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139541120, offset 0 00000400:00000200:1.0:1787695897.464547:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695897.464555:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695897.464562:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695897.464568:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695897.464574:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695897.464580:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695897.464584:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695897.464594:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17037:17037:256:4294967295] 192.168.201.140@tcp LPNI seq info [17036:17036:8:4294967295] 00000400:00000200:1.0:1787695897.464599:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695897.464612:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695897.464620:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695897.464626:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695897.464634:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695897.464637:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1787695897.464642:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695897.464645:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 37 secs later) 00000800:00000100:1.0:1787695897.464649:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695897.464652:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695897.464658:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695897.464662:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695897.464674:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695897.464677:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695897.464682:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695897.464687:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6d80 x1874531139541120/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.464697:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695897.464701:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6d80 x1874531139541120/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695922 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.464722:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695897.464724:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695897.464733:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695897.464737:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695897.464739:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298469178 00000100:00000001:1.0:1787695897.464747:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298464056 00000100:00000001:1.0:1787695897.464752:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298469178 00000100:00000001:1.0:1787695897.464756:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298464056 00000100:00000001:1.0:1787695897.464762:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695897.464765:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695897.464768:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695897.464774:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695897/real 1787695897] req@ffff8b14466f6d80 x1874531139541120/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695897.464786:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695897.464789:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695897.464791:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695897.464795:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695897.464797:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695897.464801:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695897.464805:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6d80 x1874531139541120/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695897.464816:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6d80 x1874531139541120/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.464824:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695897.464825:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.464827:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695897.464830:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695897.464836:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6d80 x1874531139541120/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.464861:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695897.464867:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6d80 x1874531139541120/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.464874:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695897.464878:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.464880:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695897.464881:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.464884:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695897.464887:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695897.464890:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695897.464895:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695897.464898:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695897.464903:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6d80 x1874531139541120/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695897.464911:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139541120:192.168.201.140@tcp:250:kworker.0 00000100:00000001:1.0:1787695897.464915:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695897.464918:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5180 x1874531139541248/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.464923:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695897.464924:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695897.464927:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695897.464929:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695897.464932:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695897.464937:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139541248:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695897.464940:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695897.464943:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695897.464945:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695897.464953:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695897.464958:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544c00. 02000000:00000001:1.0:1787695897.464960:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695897.464965:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695897.464969:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695897.464973:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695897.464977:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139541248, portal 10 00000100:00000001:1.0:1787695897.464979:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695897.464980:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930148224 : -128555779403392 : ffff8b14466f5180) 00000100:00000040:1.0:1787695897.464987:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5180 x1874531139541248/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.464994:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695897.464997:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695897.464999:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695897.465002:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139541248, offset 0 00000400:00000200:1.0:1787695897.465006:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695897.465011:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695897.465015:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695897.465018:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695897.465021:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695897.465023:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695897.465024:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695897.465027:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17038:17038:256:4294967295] 192.168.201.140@tcp LPNI seq info [17037:17037:8:4294967295] 00000400:00000200:1.0:1787695897.465029:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695897.465034:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695897.465038:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695897.465040:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695897.465043:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695897.465045:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1787695897.465049:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695897.465051:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 37 secs later) 00000800:00000100:1.0:1787695897.465054:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695897.465055:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695897.465062:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695897.465065:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695897.465067:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695897.465069:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695897.465071:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695897.465073:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5180 x1874531139541248/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465087:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695897.465089:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5180 x1874531139541248/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695922 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465094:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695897.465097:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695897.465100:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695897.465103:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000400:00000200:1.0:1787695897.465106:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695897.465108:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298459079 00000100:00000001:1.0:1787695897.465110:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298469178 00000100:00000001:1.0:1787695897.465111:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298459079 00000100:00000001:1.0:1787695897.465113:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298469178 00000100:00000001:1.0:1787695897.465114:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695897.465116:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695897.465117:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695897.465120:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695897/real 1787695897] req@ffff8b14466f5180 x1874531139541248/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695897.465125:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695897.465126:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695897.465127:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695897.465129:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695897.465130:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695897.465132:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695897.465138:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5180 x1874531139541248/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695897.465143:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5180 x1874531139541248/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465147:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695897.465148:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.465150:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695897.465150:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695897.465153:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5180 x1874531139541248/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465157:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695897.465160:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5180 x1874531139541248/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465165:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695897.465166:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.465167:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695897.465168:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.465169:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695897.465170:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695897.465172:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695897.465174:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695897.465175:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695897.465178:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5180 x1874531139541248/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695897.465182:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139541248:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695897.465185:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695897.465187:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4380 x1874531139541376/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465214:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695897.465215:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695897.465217:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695897.465218:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695897.465220:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695897.465222:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139541376:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695897.465225:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695897.465227:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695897.465228:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695897.465229:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695897.465232:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695897.465233:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695897.465236:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695897.465237:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695897.465240:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695897.465242:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139541376, portal 10 00000100:00000001:1.0:1787695897.465244:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695897.465245:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930144640 : -128555779406976 : ffff8b14466f4380) 00000100:00000040:1.0:1787695897.465249:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4380 x1874531139541376/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465255:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695897.465258:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695897.465260:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695897.465263:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139541376, offset 0 00000400:00000200:1.0:1787695897.465267:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695897.465271:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695897.465276:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695897.465280:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695897.465282:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695897.465285:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695897.465288:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695897.465296:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17039:17039:256:4294967295] 192.168.201.140@tcp LPNI seq info [17038:17038:8:4294967295] 00000400:00000200:1.0:1787695897.465301:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695897.465308:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695897.465313:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695897.465317:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695897.465321:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695897.465324:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1787695897.465328:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695897.465331:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 37 secs later) 00000800:00000100:1.0:1787695897.465335:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695897.465337:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695897.465340:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695897.465342:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695897.465344:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695897.465346:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695897.465348:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695897.465350:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4380 x1874531139541376/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465356:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695897.465358:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4380 x1874531139541376/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695922 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465402:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695897.465405:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695897.465410:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695897.465413:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787695897.465416:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695897.465418:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298457928 00000100:00000001:1.0:1787695897.465420:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298469178 00000100:00000001:1.0:1787695897.465422:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298457928 00000100:00000001:1.0:1787695897.465423:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298469178 00000100:00000001:1.0:1787695897.465425:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695897.465431:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695897.465432:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695897.465436:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695897/real 1787695897] req@ffff8b14466f4380 x1874531139541376/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695897.465442:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695897.465443:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695897.465444:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695897.465446:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695897.465447:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695897.465449:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695897.465451:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4380 x1874531139541376/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695897.465457:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4380 x1874531139541376/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465461:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695897.465462:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.465463:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695897.465464:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695897.465466:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4380 x1874531139541376/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465470:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695897.465472:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4380 x1874531139541376/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465477:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695897.465478:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.465480:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695897.465481:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.465482:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695897.465483:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695897.465485:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695897.465491:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695897.465492:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695897.465495:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4380 x1874531139541376/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695897.465500:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139541376:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695897.465503:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695897.465505:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5f80 x1874531139541504/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465510:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695897.465511:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695897.465512:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695897.465514:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695897.465515:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695897.465517:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139541504:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695897.465519:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695897.465520:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695897.465521:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695897.465522:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695897.465525:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695897.465526:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695897.465528:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695897.465530:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695897.465532:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695897.465535:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139541504, portal 4 00000100:00000001:1.0:1787695897.465536:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695897.465537:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930151808 : -128555779399808 : ffff8b14466f5f80) 00000100:00000040:1.0:1787695897.465541:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5f80 x1874531139541504/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465546:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695897.465550:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695897.465552:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695897.465554:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139541504, offset 0 00000400:00000200:1.0:1787695897.465556:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695897.465560:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695897.465562:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695897.465564:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695897.465566:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695897.465568:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695897.465569:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695897.465572:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17040:17040:256:4294967295] 192.168.201.140@tcp LPNI seq info [17039:17039:8:4294967295] 00000400:00000200:1.0:1787695897.465574:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695897.465579:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695897.465582:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695897.465584:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695897.465587:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695897.465589:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1787695897.465591:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695897.465593:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 37 secs later) 00000800:00000100:1.0:1787695897.465596:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695897.465598:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695897.465600:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695897.465602:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695897.465604:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695897.465605:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695897.465607:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695897.465609:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5f80 x1874531139541504/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465614:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695897.465616:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5f80 x1874531139541504/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695922 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465621:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695897.465622:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695897.465628:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695897.465630:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695897.465633:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695897.465634:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298457958 00000100:00000001:1.0:1787695897.465636:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298469178 00000100:00000001:1.0:1787695897.465637:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298457958 00000100:00000001:1.0:1787695897.465639:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298469178 00000100:00000001:1.0:1787695897.465640:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695897.465642:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695897.465643:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695897.465645:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695897/real 1787695897] req@ffff8b14466f5f80 x1874531139541504/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695897.465650:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695897.465651:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695897.465652:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695897.465654:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695897.465655:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695897.465656:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695897.465658:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5f80 x1874531139541504/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695897.465664:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5f80 x1874531139541504/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465669:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695897.465670:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.465672:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695897.465673:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695897.465677:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5f80 x1874531139541504/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465684:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695897.465687:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5f80 x1874531139541504/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465700:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695897.465703:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.465705:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695897.465706:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.465708:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695897.465710:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695897.465713:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695897.465717:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695897.465720:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695897.465724:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5f80 x1874531139541504/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695897.465733:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139541504:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695897.465736:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695897.465739:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5500 x1874531139541632/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465743:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695897.465744:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695897.465746:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695897.465747:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695897.465749:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695897.465750:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139541632:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695897.465752:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695897.465753:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695897.465755:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695897.465756:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695897.465758:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695897.465760:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695897.465762:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695897.465768:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695897.465770:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695897.465772:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139541632, portal 4 00000100:00000001:1.0:1787695897.465774:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695897.465775:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930149120 : -128555779402496 : ffff8b14466f5500) 00000100:00000040:1.0:1787695897.465779:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5500 x1874531139541632/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465784:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695897.465785:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695897.465787:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695897.465788:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139541632, offset 0 00000400:00000200:1.0:1787695897.465791:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695897.465794:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695897.465796:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695897.465798:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695897.465800:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695897.465801:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695897.465803:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695897.465805:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17041:17041:256:4294967295] 192.168.201.140@tcp LPNI seq info [17040:17040:8:4294967295] 00000400:00000200:1.0:1787695897.465808:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695897.465812:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695897.465815:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695897.465817:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695897.465820:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695897.465822:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1787695897.465824:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695897.465826:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 37 secs later) 00000800:00000100:1.0:1787695897.465829:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695897.465832:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695897.465835:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695897.465838:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695897.465841:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695897.465843:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695897.465851:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695897.465856:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5500 x1874531139541632/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465864:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695897.465867:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5500 x1874531139541632/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695922 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465874:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695897.465876:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695897.465880:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695897.465883:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695897.465887:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695897.465889:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298457630 00000100:00000001:1.0:1787695897.465892:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298469178 00000100:00000001:1.0:1787695897.465894:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298457630 00000100:00000001:1.0:1787695897.465896:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298469178 00000100:00000001:1.0:1787695897.465899:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695897.465900:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695897.465902:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695897.465905:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695897/real 1787695897] req@ffff8b14466f5500 x1874531139541632/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695897.465911:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695897.465912:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695897.465913:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695897.465915:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695897.465916:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695897.465918:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695897.465920:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5500 x1874531139541632/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695897.465925:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5500 x1874531139541632/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465938:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695897.465939:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.465940:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695897.465941:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695897.465944:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5500 x1874531139541632/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465948:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695897.465951:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5500 x1874531139541632/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465955:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695897.465957:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.465958:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695897.465959:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.465960:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695897.465961:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695897.465963:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695897.465965:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695897.465966:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695897.465968:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5500 x1874531139541632/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695897.465973:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139541632:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695897.465976:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695897.465979:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4700 x1874531139541760/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.465983:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695897.465984:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695897.465989:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695897.465991:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695897.465992:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695897.465994:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139541760:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695897.465996:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695897.465998:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695897.465999:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695897.466000:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695897.466003:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695897.466004:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695897.466007:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695897.466010:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695897.466013:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695897.466016:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139541760, portal 10 00000100:00000001:1.0:1787695897.466018:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695897.466020:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930145536 : -128555779406080 : ffff8b14466f4700) 00000100:00000040:1.0:1787695897.466026:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4700 x1874531139541760/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.466032:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695897.466035:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695897.466037:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695897.466040:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139541760, offset 0 00000400:00000200:1.0:1787695897.466044:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695897.466049:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695897.466052:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695897.466056:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695897.466058:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695897.466061:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695897.466064:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695897.466067:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17042:17042:256:4294967295] 192.168.201.140@tcp LPNI seq info [17041:17041:8:4294967295] 00000400:00000200:1.0:1787695897.466070:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695897.466075:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695897.466083:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695897.466086:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695897.466089:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695897.466090:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1787695897.466093:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695897.466095:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 37 secs later) 00000800:00000100:1.0:1787695897.466098:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695897.466099:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695897.466101:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695897.466104:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695897.466106:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695897.466107:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695897.466109:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695897.466111:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4700 x1874531139541760/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.466117:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695897.466119:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4700 x1874531139541760/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695922 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.466123:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695897.466124:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695897.466127:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695897.466129:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695897.466131:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695897.466133:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298457816 00000100:00000001:1.0:1787695897.466135:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298469178 00000100:00000001:1.0:1787695897.466136:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298457816 00000100:00000001:1.0:1787695897.466138:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298469178 00000100:00000001:1.0:1787695897.466139:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695897.466140:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695897.466141:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695897.466143:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695897/real 1787695897] req@ffff8b14466f4700 x1874531139541760/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695897.466153:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695897.466154:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695897.466155:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695897.466157:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695897.466158:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695897.466159:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695897.466162:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4700 x1874531139541760/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695897.466167:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4700 x1874531139541760/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.466171:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695897.466172:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.466173:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695897.466174:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695897.466176:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4700 x1874531139541760/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.466180:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695897.466183:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4700 x1874531139541760/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.466187:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695897.466188:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.466189:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695897.466205:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.466207:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695897.466208:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695897.466209:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695897.466212:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695897.466213:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695897.466220:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4700 x1874531139541760/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695897.466225:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139541760:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695897.466228:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695897.466231:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4000 x1874531139541888/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.466235:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695897.466236:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695897.466237:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695897.466239:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695897.466240:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695897.466241:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139541888:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695897.466243:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695897.466245:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695897.466246:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695897.466248:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695897.466250:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695897.466251:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695897.466254:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695897.466256:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695897.466258:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695897.466260:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139541888, portal 10 00000100:00000001:1.0:1787695897.466261:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695897.466262:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930143744 : -128555779407872 : ffff8b14466f4000) 00000100:00000040:1.0:1787695897.466266:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4000 x1874531139541888/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.466271:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695897.466272:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695897.466274:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695897.466275:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139541888, offset 0 00000400:00000200:1.0:1787695897.466281:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695897.466284:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695897.466293:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695897.466296:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695897.466300:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695897.466302:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695897.466303:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695897.466305:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17043:17043:256:4294967295] 192.168.201.140@tcp LPNI seq info [17042:17042:8:4294967295] 00000400:00000200:1.0:1787695897.466308:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695897.466313:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695897.466315:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695897.466318:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695897.466320:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695897.466322:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 37 secs later) 00000800:00000200:1.1:1787695897.466326:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695897.466328:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 37 secs later) 00000800:00000100:1.0:1787695897.466332:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695897.466334:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695897.466337:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695897.466341:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695897.466344:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695897.466346:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695897.466349:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695897.466353:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4000 x1874531139541888/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695922 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.466361:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695897.466386:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4000 x1874531139541888/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695922 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.466393:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695897.466395:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695897.466400:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695897.466403:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695897.466407:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695897.466416:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298457795 00000100:00000001:1.0:1787695897.466419:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298469178 00000100:00000001:1.0:1787695897.466421:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298457795 00000100:00000001:1.0:1787695897.466423:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298469178 00000100:00000001:1.0:1787695897.466425:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695897.466427:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695897.466429:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695897.466435:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695897/real 1787695897] req@ffff8b14466f4000 x1874531139541888/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695897.466443:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695897.466444:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695897.466446:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695897.466449:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695897.466451:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695897.466453:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695897.466457:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4000 x1874531139541888/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695897.466466:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4000 x1874531139541888/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.466473:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695897.466474:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.466475:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695897.466477:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695897.466479:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4000 x1874531139541888/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.466484:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695897.466486:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4000 x1874531139541888/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.466490:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695897.466496:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.466497:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695897.466498:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.466500:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695897.466501:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695897.466503:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695897.466505:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695897.466506:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695897.466509:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4000 x1874531139541888/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695897.466514:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139541888:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695897.466517:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.466518:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695897.466521:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6d80 x1874531139541120/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.466525:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695897.466527:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695897.466529:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545000. 02000000:00000001:1.0:1787695897.466532:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695897.466535:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695897.466536:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:1.0:1787695897.466538:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695897.466540:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674800. 02000000:00000001:1.0:1787695897.466543:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695897.466545:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695897.466546:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6d80. 00000100:00000001:1.0:1787695897.466549:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695897.466550:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.466551:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695897.466553:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5180 x1874531139541248/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.466565:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695897.466566:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695897.466568:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544c00. 02000000:00000001:1.0:1787695897.466570:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695897.466571:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695897.466572:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695897.466573:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695897.466574:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:1.0:1787695897.466575:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695897.466576:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695897.466576:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5180. 00000100:00000001:1.0:1787695897.466578:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695897.466579:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.466580:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695897.466582:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4380 x1874531139541376/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.466586:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695897.466586:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695897.466587:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695897.466589:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695897.466590:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695897.466590:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695897.466592:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695897.466593:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:1.0:1787695897.466595:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695897.466596:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695897.466596:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4380. 00000100:00000001:1.0:1787695897.466598:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695897.466599:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.466600:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695897.466601:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5f80 x1874531139541504/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.466605:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695897.466606:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695897.466609:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695897.466611:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695897.466612:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695897.466613:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695897.466614:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695897.466615:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:1.0:1787695897.466618:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695897.466619:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695897.466620:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5f80. 00000100:00000001:1.0:1787695897.466621:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695897.466622:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.466623:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695897.466625:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5500 x1874531139541632/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.466629:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695897.466629:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695897.466630:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695897.466632:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695897.466633:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695897.466634:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695897.466635:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695897.466635:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676000. 02000000:00000001:1.0:1787695897.466637:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695897.466637:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695897.466638:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5500. 00000100:00000001:1.0:1787695897.466639:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695897.466640:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.466641:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695897.466643:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4700 x1874531139541760/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.466647:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695897.466648:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695897.466649:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695897.466650:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695897.466650:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695897.466652:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695897.466655:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695897.466657:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:1.0:1787695897.466658:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695897.466659:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695897.466661:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4700. 00000100:00000001:1.0:1787695897.466663:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695897.466664:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.466666:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695897.466669:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4000 x1874531139541888/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695922 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695897.466676:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695897.466677:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695897.466679:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695897.466682:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695897.466684:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695897.466685:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695897.466686:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695897.466688:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675400. 02000000:00000001:1.0:1787695897.466691:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695897.466692:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695897.466693:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4000. 00000100:00000001:1.0:1787695897.466696:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695897.466697:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.466700:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695897.466708:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695897.466710:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695897.466713:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695897.466715:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695897.526427:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695897.526434:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695897.526436:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695897.526442:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695897.526446:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695897.526449:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695897.526452:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695897.526455:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695897.526457:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695898.166459:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695898.166465:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695898.166469:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695898.166470:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695898.166473:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695898.166474:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695898.166476:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695898.166477:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695898.166479:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695898.166495:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695898.166498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695898.166499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695898.166554:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695898.166557:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695898.166559:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695898.550450:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695898.550457:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695898.550459:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695898.550463:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695898.550466:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695898.550469:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695898.550471:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695898.550473:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695898.550474:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695899.190469:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695899.190476:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695899.190483:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695899.190485:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695899.190489:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695899.190490:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695899.190494:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695899.190496:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695899.190500:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695899.190501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695899.190504:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695899.190506:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695899.190580:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695899.190584:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695899.190587:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695899.574487:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695899.574493:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695899.574495:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695899.574499:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695899.574502:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695899.574505:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695899.574507:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695899.574509:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695899.574511:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695900.214462:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695900.214468:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695900.214473:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695900.214474:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695900.214477:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695900.214478:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695900.214480:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695900.214481:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695900.214483:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695900.214484:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695900.214487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695900.214488:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695900.214541:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695900.214545:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695900.214577:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695900.598436:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695900.598441:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695900.598443:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695900.598448:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695900.598451:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695900.598454:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695900.598456:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695900.598458:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695900.598460:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695901.238431:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695901.238436:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695901.238441:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695901.238442:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695901.238445:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695901.238446:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695901.238448:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695901.238449:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695901.238451:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695901.238454:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695901.238456:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695901.238475:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695901.238476:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695901.238477:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695901.238480:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695901.238481:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695901.238483:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695901.238485:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695901.238487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695901.238488:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695901.238532:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695901.238535:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695901.238537:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695901.622482:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695901.622491:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695901.622493:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695901.622500:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695901.622504:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695901.622508:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695901.622511:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695901.622514:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695901.622516:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695902.262462:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695902.262468:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695902.262473:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695902.262473:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695902.262476:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695902.262477:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695902.262480:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695902.262482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695902.262486:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695902.262487:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695902.262490:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695902.262492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695902.262559:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695902.262563:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695902.262566:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787695902.583717:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695902.583727:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787695902.583729:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695902.583732:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695902.583737:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695902.583743:0:47074:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3790 00000400:00000200:2.0:1787695902.583752:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695902.583756:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695902.583759:0:47074:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695902.583773:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695902.583776:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695902.583778:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695902.583779:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695902.583782:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695902.583784:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:2.0:1787695902.583786:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695902.583788:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:2.0:1787695902.583790:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:2.0:1787695902.583792:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:2.0:1787695902.583794:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695902.583795:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695902.583797:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695902.583798:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695902.583800:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:2.0:1787695902.583802:0:47074:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695902.583804:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695902.583806:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695902.583809:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695902.583811:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695902.583813:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787695902.583815:0:47074:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695902.583821:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465afb80. 00000020:00000040:2.0:1787695902.583823:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:2.0:1787695902.583826:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695902.583827:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695902.583829:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695902.583832:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695902.583836:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695902.583844:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661000. 00000100:00000001:2.0:1787695902.583852:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695902.583861:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465afb80 x1874531139542016/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695902.583878:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695902.583883:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695902.583888:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b9acb62b] to pc [ptlrpcd_rcv+-1] req@ffff8b14465afb80 x1874531139542016/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695902.583914:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695902.583917:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695902.583924:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787695902.583926:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695902.583928:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695902.583931:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695902.583933:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3790 00000400:00000200:2.0:1787695902.583938:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695902.583943:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695902.583945:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695902.583949:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1787695902.583949:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695902.583954:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695902.583955:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695902.583958:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695902.583961:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695902.583968:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465afb80 x1874531139542016/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695902.583968:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695902.583975:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695902.583978:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000100:00000001:1.0:1787695902.583983:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1787695902.583984:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695902.583986:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1787695902.583989:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 02000000:00000001:1.0:1787695902.583991:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695902.583996:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1787695902.583996:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 02000000:00000001:1.0:1787695902.584001:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695902.584004:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139542016:192.168.201.140@tcp:250:kworker.0 00000100:00000001:1.0:1787695902.584008:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1787695902.584008:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 02000000:00000001:1.0:1787695902.584012:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695902.584015:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695902.584017:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695902.584018:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787695902.584022:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695902.584027:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000010:1.0:1787695902.584029:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546000. 00000100:00000001:2.0:1787695902.584030:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:1.0:1787695902.584032:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695902.584037:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695902.584040:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695902.584045:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695902.584052:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139542016, portal 25 00000020:00000040:2.0:1787695902.584054:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00000001:1.0:1787695902.584056:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695902.584058:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928815488 : -128555780736128 : ffff8b14465afb80) 00000100:00080000:2.0:1787695902.584060:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695902.584064:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695902.584066:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465afb80 x1874531139542016/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695902.584067:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695902.584073:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695902.584076:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1787695902.584077:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695902.584082:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695902.584082:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:1.0:1787695902.584088:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000008:00100000:2.0:1787695902.584088:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000200:1.0:1787695902.584093:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139542016, offset 0 00000008:00000001:2.0:1787695902.584093:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695902.584099:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695902.584120:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000010:2.0:1787695902.584124:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ad500. 00000400:00000200:1.0:1787695902.584128:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000040:2.0:1787695902.584128:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:2.0:1787695902.584133:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695902.584135:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695902.584138:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:2.0:1787695902.584138:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695902.584140:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695902.584144:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695902.584145:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000010:2.0:1787695902.584149:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445660c00. 00000400:00000200:1.0:1787695902.584152:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:2.0:1787695902.584155:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695902.584157:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695902.584163:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00100000:2.0:1787695902.584164:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ad500 x1874531139542144/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695902.584168:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17044:17044:256:4294967295] 192.168.201.140@tcp LPNI seq info [17043:17043:8:4294967295] 00000400:00000200:1.0:1787695902.584180:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695902.584185:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695902.584188:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695902.584196:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000040:2.0:1787695902.584196:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004ab396cc] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ad500 x1874531139542144/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1787695902.584209:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695902.584216:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1787695902.584218:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000100:00080000:2.0:1787695902.584219:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695902.584223:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787695902.584225:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000200:1.0:1787695902.584228:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00080000:2.0:1787695902.584229:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.0:1787695902.584235:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 32 secs later) 00000100:00000001:2.0:1787695902.584235:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:1.1:1787695902.584243:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00080000:2.0:1787695902.584247:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3790 00000800:00000200:1.1:1787695902.584249:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 32 secs later) 00000400:00000200:2.0:1787695902.584252:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695902.584255:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695902.584258:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695902.584264:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000800:00000100:1.0:1787695902.584266:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695902.584267:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:1.0:1787695902.584271:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000100:00000040:2.0:1787695902.584273:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695902.584276:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695902.584277:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000001:2.0:1787695902.584278:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695902.584280:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:2.0:1787695902.584281:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695902.584282:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000400:00000200:1.0:1787695902.584290:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000020:00000001:2.0:1787695902.584293:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000400:00000200:1.0:1787695902.584295:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000020:00000001:2.0:1787695902.584297:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000400:00000010:1.0:1787695902.584299:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000040:2.0:1787695902.584304:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695902.584305:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787695902.584310:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1787695902.584313:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465afb80 x1874531139542016/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695902.584314:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695902.584320:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695902.584325:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:2.0:1787695902.584331:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695902.584333:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787695902.584334:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695902.584337:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1787695902.584340:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465afb80 x1874531139542016/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695927 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695902.584341:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695902.584346:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695902.584352:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695902.584353:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.584356:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000008:00100000:2.0:1787695902.584356:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787695902.584362:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.584387:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695902.584392:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695902.584395:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298474298 00000100:00000001:1.0:1787695902.584401:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298469178 00000100:00000001:1.0:1787695902.584404:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298474298 00000100:00000001:1.0:1787695902.584407:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298469178 00000100:00000001:1.0:1787695902.584410:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695902.584416:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695902.584417:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.584419:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000010:2.0:1787695902.584422:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465aca80. 00000100:00100000:1.0:1787695902.584426:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695902/real 1787695902] req@ffff8b14465afb80 x1874531139542016/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1787695902.584427:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:2.0:1787695902.584432:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000001:1.0:1787695902.584438:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1787695902.584440:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787695902.584441:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695902.584445:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000100:00000001:2.0:1787695902.584445:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695902.584449:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1787695902.584450:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 02000000:00000001:2.0:1787695902.584451:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695902.584453:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 02000000:00000010:2.0:1787695902.584456:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661400. 00000100:00000001:1.0:1787695902.584457:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695902.584462:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465afb80 x1874531139542016/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695902.584463:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695902.584472:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465aca80 x1874531139542272/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1787695902.584475:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465afb80 x1874531139542016/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695902.584480:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695902.584484:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695902.584492:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004fad31e4] to pc [ptlrpcd_rcv+-1] req@ffff8b14465aca80 x1874531139542272/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.584494:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695902.584497:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.584501:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695902.584504:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695902.584506:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695902.584509:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787695902.584511:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465afb80 x1874531139542016/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695902.584513:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787695902.584516:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695902.584519:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695902.584525:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695902.584526:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787695902.584530:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3790 00000100:00000040:1.0:1787695902.584534:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465afb80 x1874531139542016/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787695902.584538:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695902.584544:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695902.584549:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695902.584550:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:2.0:1787695902.584554:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695902.584556:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695902.584559:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695902.584560:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695902.584561:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.584563:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695902.584564:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695902.584566:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695902.584571:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695902.584574:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695902.584575:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695902.584581:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465afb80 x1874531139542016/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695902.584582:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695902.584588:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695902.584591:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000100:00100000:1.0:1787695902.584596:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139542016:192.168.201.140@tcp:250:kworker.0 00000020:00000001:2.0:1787695902.584596:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695902.584601:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000100:00000001:1.0:1787695902.584602:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.584605:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1787695902.584608:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:1.0:1787695902.584609:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465afb80 x1874531139542016/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.584614:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695902.584617:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:2.0:1787695902.584618:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:2.0:1787695902.584622:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000010:1.0:1787695902.584625:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546000. 00000100:00000001:2.0:1787695902.584626:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695902.584630:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695902.584632:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787695902.584633:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695902.584635:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695902.584637:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000020:00000040:1.0:1787695902.584638:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000100:00080000:2.0:1787695902.584642:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000020:00000001:1.0:1787695902.584643:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695902.584647:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661000. 00000100:00000001:2.0:1787695902.584647:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695902.584652:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695902.584654:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695902.584655:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695902.584657:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1787695902.584660:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695902.584661:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465afb80. 00000100:00000001:2.0:1787695902.584662:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695902.584666:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787695902.584668:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000008:00000001:2.0:1787695902.584669:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.584672:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695902.584673:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695902.584679:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ad880. 00000100:00000001:1.0:1787695902.584680:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1787695902.584683:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:2.0:1787695902.584686:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695902.584687:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695902.584688:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695902.584689:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787695902.584690:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1787695902.584691:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.584695:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695902.584696:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663000. 00000100:00000001:1.0:1787695902.584703:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695902.584703:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.584707:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1787695902.584708:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ad880 x1874531139542400/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695902.584710:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787695902.584718:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1787695902.584719:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ad500 x1874531139542144/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695902.584722:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695902.584731:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000075b4d632] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ad880 x1874531139542400/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.584740:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695902.584743:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.584746:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695902.584749:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695902.584751:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787695902.584754:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1787695902.584755:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695902.584761:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139542144:192.168.201.140@tcp:38:kworker.0 00000100:00080000:2.0:1787695902.584763:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1787695902.584766:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787695902.584768:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1787695902.584769:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695902.584772:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695902.584773:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1787695902.584774:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695902.584778:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546000. 00000100:00000001:2.0:1787695902.584780:0:47074:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1787695902.584781:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695902.584786:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3790 00000400:00000200:1.0:1787695902.584787:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695902.584792:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000200:2.0:1787695902.584796:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695902.584798:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000400:00000200:2.0:1787695902.584802:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000200:1.0:1787695902.584804:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139542144, portal 10 00000100:00080000:2.0:1787695902.584805:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695902.584808:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695902.584811:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695902.584814:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:2.0:1787695902.584815:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695902.584816:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928805632 : -128555780745984 : ffff8b14465ad500) 00000100:00000001:2.0:1787695902.584818:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695902.584821:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695902.584823:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:2.0:1787695902.584826:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000040:1.0:1787695902.584827:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ad500 x1874531139542144/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1787695902.584828:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:2.0:1787695902.584833:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:2.0:1787695902.584838:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000001:1.0:1787695902.584844:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695902.584845:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695902.584849:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695902.584850:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695902.584854:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000040:2.0:1787695902.584855:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000200:1.0:1787695902.584859:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139542144, offset 0 00000100:00000001:2.0:1787695902.584859:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695902.584865:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000400:00000200:1.0:1787695902.584869:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00080000:2.0:1787695902.584871:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695902.584874:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695902.584878:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:2.0:1787695902.584878:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695902.584883:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1787695902.584886:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 02000000:00000001:2.0:1787695902.584887:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695902.584892:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:1.0:1787695902.584893:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695902.584896:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695902.584897:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695902.584902:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000008:00100000:2.0:1787695902.584902:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787695902.584907:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695902.584908:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17045:17045:256:4294967295] 192.168.201.140@tcp LPNI seq info [17044:17044:8:4294967295] 00000100:00000001:2.0:1787695902.584913:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695902.584917:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000010:2.0:1787695902.584919:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465af800. 00000020:00000040:2.0:1787695902.584924:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 00000400:00000200:1.0:1787695902.584929:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 02000000:00000001:2.0:1787695902.584930:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695902.584933:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695902.584936:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695902.584936:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695902.584938:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695902.584941:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1787695902.584942:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 02000000:00000010:2.0:1787695902.584946:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661000. 00000800:00000200:1.0:1787695902.584947:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:2.0:1787695902.584951:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695902.584952:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 32 secs later) 00000100:00100000:2.0:1787695902.584962:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465af800 x1874531139542528/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1787695902.584962:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695902.584968:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1787695902.584977:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695902.584979:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000010:1.0:1787695902.584980:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000100:00000001:2.0:1787695902.584983:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695902.584985:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:2.0:1787695902.584990:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b410802f] to pc [ptlrpcd_rcv+-1] req@ffff8b14465af800 x1874531139542528/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695902.584992:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695902.584998:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695902.585001:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695902.585006:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787695902.585007:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695902.585010:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1787695902.585012:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ad500 x1874531139542144/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695902.585015:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787695902.585018:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695902.585020:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695902.585025:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695902.585027:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3790 00000400:00000200:2.0:1787695902.585034:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695902.585039:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695902.585044:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695902.585045:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695902.585047:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695902.585050:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695902.585052:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ad500 x1874531139542144/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695927 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695902.585054:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695902.585059:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.585063:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695902.585065:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695902.585067:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695902.585068:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1787695902.585069:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:2.0:1787695902.585072:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000040:1.0:1787695902.585073:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000020:00000040:2.0:1787695902.585074:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000020:00000001:2.0:1787695902.585079:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000200:1.0:1787695902.585082:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:2.0:1787695902.585084:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:1.0:1787695902.585086:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298464199 00000100:00000040:2.0:1787695902.585089:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695902.585091:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298474298 00000100:00000001:2.0:1787695902.585092:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695902.585096:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298464199 00000100:00000040:2.0:1787695902.585097:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695902.585114:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298474298 00000100:00000001:2.0:1787695902.585117:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695902.585119:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1787695902.585121:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00000001:1.0:1787695902.585125:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695902.585127:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695902.585128:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787695902.585130:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695902.585133:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1787695902.585136:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695902/real 1787695902] req@ffff8b14465ad500 x1874531139542144/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695902.585137:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695902.585142:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695902.585146:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695902.585149:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000001:1.0:1787695902.585154:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000001:2.0:1787695902.585154:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695902.585156:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787695902.585157:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695902.585160:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000100:00000010:2.0:1787695902.585161:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ace00. 00000400:00000200:1.0:1787695902.585163:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000020:00000040:2.0:1787695902.585165:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000400:00000010:1.0:1787695902.585167:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 02000000:00000001:2.0:1787695902.585169:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787695902.585171:0:2402:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1787695902.585171:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695902.585175:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695902.585178:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:1.0:1787695902.585179:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ad500 x1874531139542144/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695902.585179:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695902.585185:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663400. 00000100:00000001:2.0:1787695902.585190:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695902.585198:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ace00 x1874531139542656/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1787695902.585203:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ad500 x1874531139542144/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.585233:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695902.585234:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695902.585235:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695902.585239:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046eddc29] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ace00 x1874531139542656/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.585242:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.585247:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695902.585249:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695902.585253:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695902.585258:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ad500 x1874531139542144/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695902.585258:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695902.585267:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787695902.585269:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695902.585272:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695902.585277:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695902.585278:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695902.585282:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3790 00000100:00000040:1.0:1787695902.585286:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ad500 x1874531139542144/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787695902.585290:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695902.585297:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695902.585300:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695902.585301:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695902.585305:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695902.585307:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695902.585309:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695902.585311:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695902.585312:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.585315:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000040:2.0:1787695902.585316:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695902.585317:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695902.585320:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00080000:1.0:1787695902.585321:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000020:00000001:2.0:1787695902.585323:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695902.585326:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000100:00080000:1.0:1787695902.585327:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000020:00000001:2.0:1787695902.585330:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695902.585331:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000040:2.0:1787695902.585333:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:2.0:1787695902.585336:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:1.0:1787695902.585337:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ad500 x1874531139542144/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787695902.585339:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:2.0:1787695902.585347:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695902.585352:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1787695902.585356:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139542144:192.168.201.140@tcp:38:kworker.0 00000100:00000040:2.0:1787695902.585358:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695902.585361:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695902.585383:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465aca80 x1874531139542272/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695902.585384:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695902.585388:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695902.585390:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695902.585394:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695902.585397:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1787695902.585397:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 02000000:00000001:1.0:1787695902.585399:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695902.585404:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00100000:1.0:1787695902.585412:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139542272:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695902.585412:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695902.585417:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695902.585418:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695902.585420:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695902.585421:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695902.585424:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695902.585425:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695902.585428:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787695902.585429:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000010:1.0:1787695902.585432:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 00000008:00100000:2.0:1787695902.585433:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787695902.585437:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787695902.585438:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695902.585442:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695902.585443:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000010:2.0:1787695902.585447:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ae680. 00000400:00000010:1.0:1787695902.585448:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000020:00000040:2.0:1787695902.585451:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000400:00000010:1.0:1787695902.585453:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 02000000:00000001:2.0:1787695902.585455:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695902.585457:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695902.585458:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139542272, portal 10 00000100:00000001:2.0:1787695902.585459:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695902.585462:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787695902.585463:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1787695902.585465:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.585466:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928802944 : -128555780748672 : ffff8b14465aca80) 02000000:00000010:2.0:1787695902.585472:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445660400. 00000100:00000040:1.0:1787695902.585474:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465aca80 x1874531139542272/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695902.585477:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.585485:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1787695902.585486:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ae680 x1874531139542784/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1787695902.585490:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695902.585494:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695902.585500:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139542272, offset 0 00000100:00000001:2.0:1787695902.585500:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695902.585504:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695902.585509:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:2.0:1787695902.585513:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019831621] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ae680 x1874531139542784/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695902.585516:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695902.585523:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:2.0:1787695902.585527:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695902.585528:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1787695902.585531:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695902.585533:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000040:2.0:1787695902.585537:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1787695902.585538:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695902.585543:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000040:2.0:1787695902.585544:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695902.585549:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17046:17046:256:4294967295] 192.168.201.140@tcp LPNI seq info [17045:17045:8:4294967295] 00000100:00000040:2.0:1787695902.585551:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1787695902.585554:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695902.585559:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695902.585564:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695902.585568:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000100:00000040:2.0:1787695902.585571:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3800) 00000800:00000200:1.0:1787695902.585580:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695902.585583:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1787695902.585587:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695902.585589:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1787695902.585592:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695902.585593:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695902.585596:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695902.585598:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695902.585601:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695902.585602:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695902.585604:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695902.585607:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465aca80 x1874531139542272/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.585612:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695902.585614:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465aca80 x1874531139542272/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695927 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.585618:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.585619:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695902.585622:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695902.585624:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787695902.585626:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695902.585628:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298463048 00000100:00000001:1.0:1787695902.585629:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298474298 00000100:00000001:1.0:1787695902.585631:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298463048 00000100:00000001:1.0:1787695902.585632:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298474298 00000100:00000001:1.0:1787695902.585634:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695902.585635:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.585636:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695902.585638:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695902/real 1787695902] req@ffff8b14465aca80 x1874531139542272/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695902.585643:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695902.585648:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695902.585649:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695902.585650:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695902.585652:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695902.585653:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695902.585656:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465aca80 x1874531139542272/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695902.585662:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465aca80 x1874531139542272/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.585668:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695902.585670:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.585672:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695902.585674:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695902.585678:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465aca80 x1874531139542272/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.585685:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695902.585689:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465aca80 x1874531139542272/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.585696:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695902.585699:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.585701:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695902.585702:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.585704:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695902.585706:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695902.585709:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695902.585712:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695902.585713:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695902.585716:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465aca80 x1874531139542272/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695902.585731:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139542272:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695902.585734:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.585735:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695902.585738:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ad500 x1874531139542144/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.585742:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695902.585743:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695902.585745:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695902.585747:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695902.585748:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695902.585749:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695902.585750:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695902.585752:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445660c00. 02000000:00000001:1.0:1787695902.585753:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695902.585754:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695902.585756:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ad500. 00000100:00000001:1.0:1787695902.585758:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695902.585758:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.585759:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695902.585761:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465aca80 x1874531139542272/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.585765:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695902.585766:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695902.585767:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695902.585769:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695902.585770:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695902.585770:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695902.585771:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695902.585772:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661400. 02000000:00000001:1.0:1787695902.585773:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695902.585774:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695902.585775:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465aca80. 00000100:00000001:1.0:1787695902.585780:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695902.585781:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.585782:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.585785:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695902.585786:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.585788:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695902.585789:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695902.585790:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695902.585793:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ad880 x1874531139542400/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.585797:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695902.585798:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695902.585800:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695902.585801:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695902.585802:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695902.585804:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139542400:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695902.585806:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695902.585807:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695902.585809:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695902.585810:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695902.585812:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695902.585813:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695902.585816:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695902.585818:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695902.585821:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695902.585823:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139542400, portal 4 00000100:00000001:1.0:1787695902.585824:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695902.585825:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928806528 : -128555780745088 : ffff8b14465ad880) 00000100:00000040:1.0:1787695902.585841:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ad880 x1874531139542400/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.585846:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695902.585848:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695902.585849:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695902.585851:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139542400, offset 0 00000400:00000200:1.0:1787695902.585858:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695902.585861:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695902.585864:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695902.585866:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695902.585868:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695902.585869:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695902.585871:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695902.585873:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17047:17047:256:4294967295] 192.168.201.140@tcp LPNI seq info [17046:17046:8:4294967295] 00000400:00000200:1.0:1787695902.585876:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695902.585880:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695902.585883:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695902.585885:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695902.585887:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695902.585889:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1787695902.585892:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695902.585894:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1787695902.585896:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695902.585898:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695902.585900:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695902.585902:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695902.585904:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695902.585905:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695902.585907:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695902.585909:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ad880 x1874531139542400/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.585915:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695902.585917:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ad880 x1874531139542400/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695927 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.585922:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.585923:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695902.585926:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695902.585928:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695902.585930:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695902.585936:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298463078 00000100:00000001:1.0:1787695902.585938:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298474298 00000100:00000001:1.0:1787695902.585940:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298463078 00000100:00000001:1.0:1787695902.585942:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298474298 00000100:00000001:1.0:1787695902.585944:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695902.585945:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.585946:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695902.585950:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695902/real 1787695902] req@ffff8b14465ad880 x1874531139542400/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695902.585956:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695902.585957:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695902.585958:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695902.585960:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695902.585961:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695902.585962:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695902.585965:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ad880 x1874531139542400/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695902.585970:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ad880 x1874531139542400/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.585974:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695902.585975:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.585976:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695902.585977:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695902.585979:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ad880 x1874531139542400/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.585983:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695902.585985:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ad880 x1874531139542400/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.585990:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695902.585994:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.585995:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695902.585996:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.585998:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695902.585999:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695902.586001:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695902.586003:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695902.586004:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695902.586007:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ad880 x1874531139542400/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695902.586012:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139542400:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695902.586015:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695902.586018:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465af800 x1874531139542528/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586022:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695902.586023:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695902.586024:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695902.586025:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695902.586027:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695902.586028:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139542528:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695902.586030:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695902.586031:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695902.586033:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695902.586034:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695902.586036:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695902.586037:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695902.586039:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695902.586040:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695902.586042:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695902.586045:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139542528, portal 4 00000100:00000001:1.0:1787695902.586050:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695902.586050:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928814592 : -128555780737024 : ffff8b14465af800) 00000100:00000040:1.0:1787695902.586055:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465af800 x1874531139542528/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586059:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695902.586061:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695902.586062:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695902.586064:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139542528, offset 0 00000400:00000200:1.0:1787695902.586066:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695902.586069:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695902.586071:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695902.586073:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695902.586074:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695902.586076:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695902.586077:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695902.586079:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17048:17048:256:4294967295] 192.168.201.140@tcp LPNI seq info [17047:17047:8:4294967295] 00000400:00000200:1.0:1787695902.586082:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695902.586086:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695902.586089:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695902.586091:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695902.586094:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695902.586096:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1787695902.586098:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695902.586100:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1787695902.586117:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695902.586120:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695902.586123:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695902.586125:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695902.586127:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695902.586129:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695902.586130:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695902.586133:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465af800 x1874531139542528/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586147:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695902.586149:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465af800 x1874531139542528/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695927 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586153:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.586155:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695902.586157:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695902.586159:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695902.586162:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695902.586163:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298462750 00000100:00000001:1.0:1787695902.586165:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298474298 00000100:00000001:1.0:1787695902.586167:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298462750 00000100:00000001:1.0:1787695902.586169:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298474298 00000100:00000001:1.0:1787695902.586172:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695902.586174:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.586176:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695902.586180:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695902/real 1787695902] req@ffff8b14465af800 x1874531139542528/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695902.586188:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695902.586189:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695902.586191:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695902.586194:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695902.586196:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695902.586198:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695902.586201:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465af800 x1874531139542528/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695902.586211:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465af800 x1874531139542528/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586218:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695902.586223:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.586225:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695902.586226:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695902.586229:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465af800 x1874531139542528/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586233:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695902.586236:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465af800 x1874531139542528/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586241:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695902.586242:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.586243:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695902.586244:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.586245:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695902.586246:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695902.586247:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695902.586249:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695902.586251:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695902.586253:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465af800 x1874531139542528/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695902.586258:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139542528:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695902.586261:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695902.586263:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ace00 x1874531139542656/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586267:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695902.586268:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695902.586270:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695902.586271:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695902.586272:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695902.586277:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139542656:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695902.586279:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695902.586281:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695902.586282:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695902.586283:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695902.586285:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695902.586286:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695902.586289:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695902.586291:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695902.586293:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695902.586295:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139542656, portal 10 00000100:00000001:1.0:1787695902.586297:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695902.586298:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928803840 : -128555780747776 : ffff8b14465ace00) 00000100:00000040:1.0:1787695902.586303:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ace00 x1874531139542656/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586309:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695902.586311:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695902.586314:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695902.586316:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139542656, offset 0 00000400:00000200:1.0:1787695902.586321:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695902.586325:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695902.586328:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695902.586331:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695902.586334:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695902.586336:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695902.586339:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695902.586342:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17049:17049:256:4294967295] 192.168.201.140@tcp LPNI seq info [17048:17048:8:4294967295] 00000400:00000200:1.0:1787695902.586347:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695902.586354:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695902.586358:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695902.586362:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695902.586386:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695902.586388:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1787695902.586392:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695902.586394:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1787695902.586397:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695902.586399:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695902.586401:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695902.586404:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695902.586406:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695902.586407:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695902.586409:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695902.586412:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ace00 x1874531139542656/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586417:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695902.586419:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ace00 x1874531139542656/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695927 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586424:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.586425:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695902.586427:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695902.586429:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695902.586432:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695902.586433:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298462937 00000100:00000001:1.0:1787695902.586435:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298474298 00000100:00000001:1.0:1787695902.586437:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298462937 00000100:00000001:1.0:1787695902.586439:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298474298 00000100:00000001:1.0:1787695902.586440:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695902.586441:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.586443:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695902.586445:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695902/real 1787695902] req@ffff8b14465ace00 x1874531139542656/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695902.586450:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695902.586454:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695902.586455:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695902.586457:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695902.586458:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695902.586460:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695902.586462:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ace00 x1874531139542656/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695902.586467:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ace00 x1874531139542656/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586471:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695902.586472:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.586473:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695902.586474:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695902.586476:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ace00 x1874531139542656/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586481:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695902.586483:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ace00 x1874531139542656/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586488:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695902.586489:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.586490:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695902.586491:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.586492:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695902.586493:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695902.586495:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695902.586496:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695902.586498:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695902.586500:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ace00 x1874531139542656/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695902.586512:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139542656:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695902.586514:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695902.586517:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ae680 x1874531139542784/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586521:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695902.586522:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695902.586523:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695902.586525:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695902.586526:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695902.586528:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139542784:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695902.586529:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695902.586531:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695902.586532:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695902.586533:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695902.586535:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695902.586537:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695902.586539:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695902.586541:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695902.586543:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695902.586545:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139542784, portal 10 00000100:00000001:1.0:1787695902.586546:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695902.586547:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928810112 : -128555780741504 : ffff8b14465ae680) 00000100:00000040:1.0:1787695902.586551:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ae680 x1874531139542784/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586556:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695902.586557:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695902.586558:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695902.586560:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139542784, offset 0 00000400:00000200:1.0:1787695902.586562:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695902.586569:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695902.586572:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695902.586574:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695902.586575:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695902.586577:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695902.586579:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695902.586582:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17050:17050:256:4294967295] 192.168.201.140@tcp LPNI seq info [17049:17049:8:4294967295] 00000400:00000200:1.0:1787695902.586586:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695902.586592:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695902.586597:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695902.586600:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695902.586604:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695902.586607:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 32 secs later) 00000800:00000200:1.1:1787695902.586611:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695902.586614:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 32 secs later) 00000800:00000100:1.0:1787695902.586617:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695902.586620:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695902.586623:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695902.586626:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695902.586629:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695902.586631:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695902.586633:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695902.586637:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ae680 x1874531139542784/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695927 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586645:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695902.586648:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ae680 x1874531139542784/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695927 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586653:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.586654:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695902.586657:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695902.586659:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695902.586663:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695902.586670:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298462915 00000100:00000001:1.0:1787695902.586673:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298474298 00000100:00000001:1.0:1787695902.586675:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298462915 00000100:00000001:1.0:1787695902.586677:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298474298 00000100:00000001:1.0:1787695902.586679:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695902.586681:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.586683:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695902.586688:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695902/real 1787695902] req@ffff8b14465ae680 x1874531139542784/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695902.586697:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695902.586698:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695902.586700:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695902.586702:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695902.586704:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695902.586707:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695902.586711:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ae680 x1874531139542784/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695902.586718:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ae680 x1874531139542784/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586723:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695902.586724:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.586725:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695902.586727:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695902.586729:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ae680 x1874531139542784/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586733:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695902.586736:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ae680 x1874531139542784/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586740:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695902.586746:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.586747:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695902.586748:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.586749:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695902.586750:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695902.586752:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695902.586755:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695902.586756:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695902.586759:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ae680 x1874531139542784/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695902.586764:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139542784:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695902.586767:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.586768:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695902.586770:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ad880 x1874531139542400/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586774:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695902.586775:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695902.586776:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695902.586778:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695902.586779:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695902.586780:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695902.586782:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695902.586783:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663000. 02000000:00000001:1.0:1787695902.586785:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695902.586786:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695902.586787:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ad880. 00000100:00000001:1.0:1787695902.586788:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695902.586789:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.586790:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695902.586792:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465af800 x1874531139542528/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586803:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695902.586804:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695902.586806:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695902.586808:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695902.586809:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695902.586810:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695902.586812:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695902.586814:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661000. 02000000:00000001:1.0:1787695902.586816:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695902.586818:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695902.586819:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465af800. 00000100:00000001:1.0:1787695902.586821:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695902.586823:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.586824:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695902.586829:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ace00 x1874531139542656/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586836:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695902.586837:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695902.586838:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695902.586841:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695902.586842:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695902.586844:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695902.586846:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695902.586847:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663400. 02000000:00000001:1.0:1787695902.586849:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695902.586850:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695902.586852:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ace00. 00000100:00000001:1.0:1787695902.586855:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695902.586856:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.586858:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695902.586861:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ae680 x1874531139542784/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695927 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695902.586867:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695902.586869:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695902.586876:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695902.586880:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695902.586881:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695902.586883:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695902.586885:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695902.586886:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445660400. 02000000:00000001:1.0:1787695902.586888:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695902.586890:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695902.586891:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ae680. 00000100:00000001:1.0:1787695902.586893:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695902.586894:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.586895:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695902.586898:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695902.586899:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695902.586901:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695902.586902:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695902.646427:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695902.646433:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695902.646434:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695902.646438:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695902.646441:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695902.646444:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695902.646446:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695902.646448:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695902.646450:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695903.286472:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695903.286478:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695903.286481:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695903.286484:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695903.286488:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695903.286490:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695903.286492:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695903.286493:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695903.286496:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695903.286498:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695903.286498:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695903.286500:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695903.286501:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695903.286502:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695903.286503:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695903.286504:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695903.286506:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695903.286507:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695903.286510:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695903.286511:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695903.286513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695903.286513:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695903.286531:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695903.286532:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695903.286582:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695903.286585:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695903.286588:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695903.670510:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695903.670520:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695903.670523:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695903.670530:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695903.670534:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695903.670539:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695903.670542:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695903.670568:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695903.670572:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695904.310472:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695904.310480:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695904.310487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695904.310488:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695904.310493:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695904.310494:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695904.310499:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695904.310500:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695904.310503:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695904.310505:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695904.310508:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695904.310509:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695904.310608:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695904.310615:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695904.310619:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695904.694432:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695904.694437:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695904.694439:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695904.694443:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695904.694446:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695904.694449:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695904.694451:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695904.694452:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695904.694454:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695905.334453:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695905.334461:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695905.334468:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695905.334470:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695905.334474:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695905.334475:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695905.334479:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695905.334480:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695905.334484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695905.334485:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695905.334488:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695905.334490:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695905.334534:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695905.334539:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695905.334543:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695905.718450:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695905.718456:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695905.718458:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695905.718462:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695905.718464:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695905.718467:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695905.718469:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695905.718471:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695905.718473:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695906.358446:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695906.358452:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695906.358456:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695906.358457:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695906.358460:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695906.358463:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695906.358465:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695906.358470:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695906.358472:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695906.358473:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695906.358476:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695906.358478:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695906.358479:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695906.358495:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695906.358498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695906.358499:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695906.358500:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695906.358502:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695906.358503:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695906.358503:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695906.358505:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695906.358505:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695906.358506:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695906.358508:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695906.358510:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695906.358511:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695906.358513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695906.358514:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695906.358514:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695906.358516:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695906.358517:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695906.358517:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695906.358518:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695906.358519:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695906.358520:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695906.358521:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695906.358573:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695906.358576:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695906.358579:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695906.742413:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695906.742419:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695906.742421:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695906.742425:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695906.742428:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695906.742431:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695906.742432:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695906.742434:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695906.742436:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695907.382472:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695907.382478:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695907.382483:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695907.382484:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695907.382487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695907.382488:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695907.382490:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695907.382491:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695907.382493:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695907.382494:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695907.382496:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695907.382497:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695907.382551:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695907.382554:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695907.382571:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787695907.702820:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695907.702830:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787695907.702833:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695907.702837:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695907.702842:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695907.702860:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3795 00000400:00000200:3.0:1787695907.702870:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695907.702877:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695907.702880:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695907.702884:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695907.702887:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695907.702890:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695907.702892:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695907.702896:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695907.702899:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:3.0:1787695907.702902:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695907.702905:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787695907.702908:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787695907.702910:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787695907.702914:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695907.702916:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695907.702918:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695907.702920:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695907.702924:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787695907.702927:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695907.702930:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695907.702933:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695907.702937:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695907.702940:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695907.702942:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787695907.702945:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695907.702952:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4a80. 00000020:00000040:3.0:1787695907.702955:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787695907.702959:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695907.702961:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695907.702964:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695907.702967:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695907.702978:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695907.702986:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675800. 00000100:00000001:3.0:1787695907.702994:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695907.703010:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4a80 x1874531139542912/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695907.703039:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695907.703044:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695907.703050:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ad0d67] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4a80 x1874531139542912/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695907.703072:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695907.703076:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695907.703079:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695907.703081:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695907.703083:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695907.703086:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695907.703089:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3795 00000400:00000200:3.0:1787695907.703094:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695907.703099:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695907.703102:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787695907.703103:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695907.703108:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1787695907.703109:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695907.703111:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787695907.703113:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695907.703120:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4a80 x1874531139542912/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695907.703120:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695907.703123:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695907.703126:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695907.703127:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:3.0:1787695907.703131:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695907.703135:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:3.0:1787695907.703140:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000001:1.0:1787695907.703142:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695907.703147:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695907.703147:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 02000000:00000001:1.0:1787695907.703152:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695907.703156:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1787695907.703157:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695907.703158:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695907.703162:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139542912:192.168.201.140@tcp:250:kworker.0 00000100:00000001:1.0:1787695907.703166:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695907.703171:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787695907.703171:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695907.703176:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695907.703178:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695907.703181:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 02000000:00000001:1.0:1787695907.703183:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695907.703187:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000001:1.0:1787695907.703189:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787695907.703192:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695907.703197:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:1.0:1787695907.703202:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547c00. 02000000:00000001:3.0:1787695907.703203:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695907.703207:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695907.703209:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695907.703213:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:1.0:1787695907.703217:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000008:00100000:3.0:1787695907.703217:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000010:1.0:1787695907.703223:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000008:00000001:3.0:1787695907.703223:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695907.703225:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695907.703232:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5880. 00000400:00000010:1.0:1787695907.703234:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000020:00000040:3.0:1787695907.703236:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695907.703241:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000200:1.0:1787695907.703243:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139542912, portal 25 02000000:00000001:3.0:1787695907.703243:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.703250:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787695907.703250:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787695907.703254:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930146432 : -128555779405184 : ffff8b14466f4a80) 02000000:00000001:3.0:1787695907.703255:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695907.703259:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695907.703267:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4a80 x1874531139542912/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1787695907.703268:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675400. 00000100:00000001:3.0:1787695907.703275:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.703282:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695907.703287:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00100000:3.0:1787695907.703288:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5880 x1874531139543040/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1787695907.703290:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695907.703296:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139542912, offset 0 00000400:00000200:1.0:1787695907.703306:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695907.703307:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695907.703311:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695907.703315:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695907.703320:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000827e7dbe] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5880 x1874531139543040/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695907.703323:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695907.703330:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695907.703336:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695907.703342:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1787695907.703343:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695907.703345:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00080000:3.0:1787695907.703348:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695907.703355:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:1.0:1787695907.703356:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17051:17051:256:4294967295] 192.168.201.140@tcp LPNI seq info [17050:17050:8:4294967295] 00000400:00000200:1.0:1787695907.703360:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695907.703395:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:3.0:1787695907.703400:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695907.703404:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.0:1787695907.703408:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695907.703410:0:38:0:(import.c:485:import_select_connection()) Process entered 00000800:00000010:1.0:1787695907.703416:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000100:00080000:3.0:1787695907.703417:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3795 00000400:00000200:3.0:1787695907.703427:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000800:00000200:1.0:1787695907.703431:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000400:00000200:3.0:1787695907.703435:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000800:00000200:1.0:1787695907.703439:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000100:00080000:3.0:1787695907.703440:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695907.703448:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000800:00000200:1.1:1787695907.703453:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787695907.703454:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.1:1787695907.703459:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000100:00000040:3.0:1787695907.703460:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000100:1.0:1787695907.703467:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695907.703468:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000010:1.0:1787695907.703471:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695907.703475:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695907.703479:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695907.703484:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000020:00000001:3.0:1787695907.703484:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695907.703488:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000400:00000010:1.0:1787695907.703489:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000020:00000001:3.0:1787695907.703494:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695907.703496:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:3.0:1787695907.703500:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000100:00000200:1.0:1787695907.703502:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4a80 x1874531139542912/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695907.703507:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:3.0:1787695907.703512:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:3.0:1787695907.703516:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695907.703519:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695907.703522:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695907.703526:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787695907.703526:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695907.703531:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00000040:1.0:1787695907.703533:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4a80 x1874531139542912/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695932 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695907.703539:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695907.703542:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.703545:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787695907.703545:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695907.703550:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695907.703552:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695907.703555:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1787695907.703559:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787695907.703562:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.703563:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298479417 00000100:00000001:3.0:1787695907.703567:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695907.703571:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787695907.703572:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298474298 00000008:00000001:3.0:1787695907.703577:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.703579:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298479417 00000100:00000001:1.0:1787695907.703583:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298474298 00000100:00000001:1.0:1787695907.703585:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695907.703588:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695907.703590:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695907.703595:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6680. 00000100:00000001:1.0:1787695907.703597:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000040:3.0:1787695907.703601:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000100:00100000:1.0:1787695907.703604:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695907/real 1787695907] req@ffff8b14466f4a80 x1874531139542912/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695907.703606:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695907.703609:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695907.703615:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:1.0:1787695907.703619:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1787695907.703619:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695907.703621:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787695907.703623:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695907.703627:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 02000000:00000010:3.0:1787695907.703629:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 00000400:00000200:1.0:1787695907.703631:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695907.703636:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:3.0:1787695907.703637:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.703640:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:3.0:1787695907.703647:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6680 x1874531139543168/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1787695907.703649:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4a80 x1874531139542912/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695907.703676:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000200:1.0:1787695907.703679:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4a80 x1874531139542912/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695907.703681:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695907.703692:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f6d2abf9] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6680 x1874531139543168/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.703694:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695907.703698:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.703704:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695907.703707:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695907.703709:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695907.703714:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787695907.703717:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4a80 x1874531139542912/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695907.703719:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787695907.703723:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695907.703726:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695907.703732:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695907.703736:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695907.703739:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3795 00000100:00000040:1.0:1787695907.703744:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4a80 x1874531139542912/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695907.703749:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695907.703756:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695907.703758:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787695907.703761:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695907.703762:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.703767:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695907.703769:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695907.703770:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695907.703772:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695907.703775:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787695907.703776:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:1.0:1787695907.703779:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:3.0:1787695907.703781:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00080000:1.0:1787695907.703785:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:3.0:1787695907.703786:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695907.703790:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695907.703794:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695907.703799:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4a80 x1874531139542912/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1787695907.703801:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:3.0:1787695907.703805:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695907.703810:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000100:00100000:1.0:1787695907.703812:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139542912:192.168.201.140@tcp:250:kworker.0 00000020:00000001:3.0:1787695907.703815:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:1.0:1787695907.703818:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787695907.703821:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:1.0:1787695907.703822:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695907.703825:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4a80 x1874531139542912/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695907.703832:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695907.703833:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695907.703837:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1787695907.703838:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000010:1.0:1787695907.703840:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695907.703845:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:3.0:1787695907.703845:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000020:00000001:1.0:1787695907.703848:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695907.703851:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000100:00000001:3.0:1787695907.703852:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695907.703854:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695907.703858:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675800. 00000020:00000040:3.0:1787695907.703859:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 02000000:00000001:1.0:1787695907.703863:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695907.703866:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:3.0:1787695907.703866:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000010:1.0:1787695907.703869:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4a80. 00000100:00000001:3.0:1787695907.703873:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.703874:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695907.703876:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.703879:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787695907.703879:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695907.703886:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695907.703888:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1787695907.703890:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.703891:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.703895:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695907.703898:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695907.703898:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695907.703902:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:3.0:1787695907.703903:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000040:1.0:1787695907.703906:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5880 x1874531139543040/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1787695907.703911:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.703917:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695907.703917:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.703920:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695907.703923:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695907.703924:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1787695907.703924:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5c00. 00000020:00000040:3.0:1787695907.703929:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:1.0:1787695907.703932:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695907.703936:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1787695907.703937:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139543040:192.168.201.140@tcp:38:kworker.0 02000000:00000001:3.0:1787695907.703940:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.703942:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787695907.703945:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695907.703946:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695907.703950:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695907.703950:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695907.703954:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787695907.703956:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695907.703957:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695907.703963:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695907.703967:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000010:3.0:1787695907.703967:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676000. 00000400:00000010:1.0:1787695907.703972:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000100:00000001:3.0:1787695907.703974:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695907.703976:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695907.703980:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139543040, portal 10 00000100:00000001:1.0:1787695907.703984:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695907.703985:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150016 : -128555779401600 : ffff8b14466f5880) 00000100:00100000:3.0:1787695907.703985:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5c00 x1874531139543296/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1787695907.703995:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5880 x1874531139543040/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695907.704003:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695907.704005:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787695907.704009:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695907.704010:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695907.704012:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695907.704032:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139543040, offset 0 00000400:00000200:1.0:1787695907.704041:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695907.704044:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005bfbcee1] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5c00 x1874531139543296/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695907.704047:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695907.704053:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695907.704060:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695907.704065:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695907.704068:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1787695907.704070:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695907.704071:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695907.704077:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17052:17052:256:4294967295] 192.168.201.140@tcp LPNI seq info [17051:17051:8:4294967295] 00000100:00080000:3.0:1787695907.704077:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695907.704084:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:3.0:1787695907.704088:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1787695907.704093:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1787695907.704095:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00080000:3.0:1787695907.704097:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.0:1787695907.704102:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695907.704105:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695907.704110:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3795 00000800:00000010:1.0:1787695907.704111:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000400:00000200:3.0:1787695907.704120:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000800:00000200:1.0:1787695907.704121:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695907.704127:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000400:00000200:3.0:1787695907.704131:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000800:00000200:1.1:1787695907.704134:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00080000:3.0:1787695907.704137:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000800:00000200:1.1:1787695907.704138:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1787695907.704148:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695907.704148:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000800:00000010:1.0:1787695907.704150:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000100:00000001:3.0:1787695907.704154:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695907.704155:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695907.704162:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00000040:3.0:1787695907.704162:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695907.704168:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000100:00000001:3.0:1787695907.704169:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000010:1.0:1787695907.704171:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000020:00000001:3.0:1787695907.704175:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695907.704176:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:3.0:1787695907.704179:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000100:00000200:1.0:1787695907.704183:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5880 x1874531139543040/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695907.704183:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695907.704186:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:3.0:1787695907.704191:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:3.0:1787695907.704196:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:3.0:1787695907.704205:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695907.704206:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787695907.704208:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695907.704214:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695907.704215:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5880 x1874531139543040/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695932 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695907.704218:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695907.704223:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:3.0:1787695907.704230:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695907.704234:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695907.704235:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.704237:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695907.704241:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695907.704246:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 02000000:00000001:3.0:1787695907.704246:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1787695907.704253:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787695907.704253:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695907.704256:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298469318 02000000:00000001:3.0:1787695907.704259:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.704260:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298479417 00000100:00000001:1.0:1787695907.704265:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298469318 00000100:00000001:3.0:1787695907.704265:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695907.704270:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298479417 00000008:00100000:3.0:1787695907.704271:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787695907.704272:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695907.704276:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787695907.704279:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.704280:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787695907.704284:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695907.704285:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695907/real 1787695907] req@ffff8b14466f5880 x1874531139543040/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1787695907.704290:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6a00. 00000020:00000040:3.0:1787695907.704295:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:3.0:1787695907.704298:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695907.704301:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695907.704306:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695907.704309:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:1.0:1787695907.704312:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1787695907.704313:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787695907.704315:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695907.704318:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695907.704321:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 02000000:00000010:3.0:1787695907.704321:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 00000400:00000010:1.0:1787695907.704324:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695907.704328:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787695907.704329:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695907.704332:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5880 x1874531139543040/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695907.704340:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6a00 x1874531139543424/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1787695907.704344:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5880 x1874531139543040/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.704379:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695907.704380:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.704382:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695907.704383:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695907.704386:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5880 x1874531139543040/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695907.704392:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695907.704397:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.704404:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695907.704408:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002ad47e8f] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6a00 x1874531139543424/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695907.704414:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5880 x1874531139543040/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695907.704427:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.704434:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787695907.704434:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695907.704441:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695907.704442:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787695907.704446:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787695907.704447:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787695907.704448:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.704451:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1787695907.704451:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695907.704455:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787695907.704458:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695907.704459:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695907.704465:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00080000:3.0:1787695907.704465:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3795 00000100:00000001:1.0:1787695907.704473:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000400:00000200:3.0:1787695907.704476:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695907.704483:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000040:1.0:1787695907.704487:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5880 x1874531139543040/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695907.704488:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787695907.704494:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139543040:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695907.704501:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1787695907.704504:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695907.704506:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6680 x1874531139543168/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695907.704509:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695907.704516:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695907.704517:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695907.704520:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695907.704522:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:1.0:1787695907.704523:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695907.704527:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1787695907.704528:0:38:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1787695907.704531:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695907.704533:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000100:00100000:1.0:1787695907.704537:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139543168:192.168.201.140@tcp:38:kworker.0 00000020:00000001:3.0:1787695907.704538:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695907.704543:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1787695907.704543:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 02000000:00000001:1.0:1787695907.704545:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695907.704548:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695907.704548:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 02000000:00000001:1.0:1787695907.704553:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1787695907.704555:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 02000000:00000010:1.0:1787695907.704557:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695907.704562:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695907.704562:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695907.704567:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695907.704568:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695907.704573:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000100:00000040:3.0:1787695907.704575:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000010:1.0:1787695907.704577:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000001:3.0:1787695907.704579:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000200:1.0:1787695907.704582:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139543168, portal 10 00000020:00000040:3.0:1787695907.704583:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00000001:1.0:1787695907.704585:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695907.704589:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930153600 : -128555779398016 : ffff8b14466f6680) 00000100:00080000:3.0:1787695907.704591:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695907.704595:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695907.704596:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6680 x1874531139543168/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695907.704600:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695907.704606:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787695907.704607:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1787695907.704611:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695907.704613:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695907.704615:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000001:3.0:1787695907.704618:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000200:1.0:1787695907.704620:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139543168, offset 0 00000008:00100000:3.0:1787695907.704622:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695907.704626:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695907.704629:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695907.704636:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7b80. 00000400:00000200:1.0:1787695907.704637:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000020:00000040:3.0:1787695907.704642:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000400:00000200:1.0:1787695907.704647:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787695907.704649:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695907.704652:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695907.704656:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:3.0:1787695907.704656:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695907.704660:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695907.704663:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695907.704664:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695907.704668:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695907.704671:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 02000000:00000010:3.0:1787695907.704671:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 00000400:00000200:1.0:1787695907.704674:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695907.704676:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17053:17053:256:4294967295] 192.168.201.140@tcp LPNI seq info [17052:17052:8:4294967295] 00000400:00000200:1.0:1787695907.704685:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695907.704685:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695907.704696:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7b80 x1874531139543552/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695907.704701:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695907.704717:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695907.704719:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695907.704723:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1787695907.704724:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000100:00000040:3.0:1787695907.704730:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b87e1da] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7b80 x1874531139543552/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1787695907.704735:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695907.704740:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1787695907.704748:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787695907.704752:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1787695907.704753:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1787695907.704759:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00080000:3.0:1787695907.704759:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:1.0:1787695907.704763:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000100:00080000:3.0:1787695907.704767:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:1.0:1787695907.704769:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1787695907.704771:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1787695907.704775:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00080000:3.0:1787695907.704775:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695907.704783:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1787695907.704784:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695907.704787:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00080000:3.0:1787695907.704790:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3795 00000100:00000001:1.0:1787695907.704792:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695907.704798:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6680 x1874531139543168/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695907.704800:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695907.704808:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695907.704809:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1787695907.704814:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695907.704815:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6680 x1874531139543168/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695932 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695907.704824:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695907.704827:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695907.704829:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695907.704831:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1787695907.704835:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695907.704837:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695907.704838:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000040:1.0:1787695907.704843:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000020:00000001:3.0:1787695907.704844:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695907.704846:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000020:00000001:3.0:1787695907.704850:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:1.0:1787695907.704853:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000040:3.0:1787695907.704854:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:3.0:1787695907.704857:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000020:00000001:3.0:1787695907.704859:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:1.0:1787695907.704866:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298468167 00000100:00000040:3.0:1787695907.704867:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695907.704873:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298479417 00000100:00000001:3.0:1787695907.704874:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695907.704879:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298468167 00000100:00000001:1.0:1787695907.704883:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298479417 00000100:00000040:3.0:1787695907.704886:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695907.704887:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695907.704891:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695907.704893:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695907.704894:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000040:3.0:1787695907.704900:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00100000:1.0:1787695907.704901:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695907/real 1787695907] req@ffff8b14466f6680 x1874531139543168/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695907.704908:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000001:1.0:1787695907.704912:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787695907.704914:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787695907.704915:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1787695907.704919:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:1.0:1787695907.704920:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695907.704925:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 02000000:00000001:3.0:1787695907.704926:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1787695907.704928:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 02000000:00000001:3.0:1787695907.704930:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.704933:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787695907.704935:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695907.704939:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000200:1.0:1787695907.704940:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6680 x1874531139543168/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1787695907.704944:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695907.704949:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695907.704956:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7800. 00000020:00000040:3.0:1787695907.704961:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000200:1.0:1787695907.704962:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6680 x1874531139543168/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695907.704967:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695907.704970:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695907.704974:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695907.704977:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787695907.704979:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:3.0:1787695907.704980:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.704983:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1787695907.704987:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 00000100:00000001:1.0:1787695907.704988:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695907.704992:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695907.704994:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695907.704999:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6680 x1874531139543168/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695907.705005:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7800 x1874531139543680/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695907.705030:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695907.705039:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1787695907.705041:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6680 x1874531139543168/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695907.705044:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695907.705054:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7c57981] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7800 x1874531139543680/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.705055:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695907.705064:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.705068:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695907.705069:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.705072:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787695907.705073:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.705074:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695907.705080:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00000040:3.0:1787695907.705080:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695907.705084:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695907.705089:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1787695907.705089:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695907.705096:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1787695907.705097:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6680 x1874531139543168/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695907.705103:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139543168:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695907.705110:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695907.705113:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1787695907.705114:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695907.705118:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5880 x1874531139543040/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695907.705123:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3805) 00000100:00000001:1.0:1787695907.705125:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695907.705128:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695907.705130:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695907.705132:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695907.705133:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695907.705135:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695907.705136:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695907.705138:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675400. 02000000:00000001:1.0:1787695907.705140:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695907.705141:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695907.705142:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5880. 00000100:00000001:1.0:1787695907.705144:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695907.705145:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.705146:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695907.705148:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6680 x1874531139543168/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.705160:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695907.705161:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695907.705162:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695907.705165:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695907.705165:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695907.705166:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695907.705167:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695907.705168:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:1.0:1787695907.705171:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695907.705172:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695907.705173:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6680. 00000100:00000001:1.0:1787695907.705174:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695907.705175:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.705176:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.705179:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695907.705180:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.705182:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695907.705183:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695907.705185:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695907.705189:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5c00 x1874531139543296/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.705197:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695907.705198:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695907.705201:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695907.705203:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695907.705205:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695907.705208:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139543296:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695907.705212:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695907.705214:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695907.705215:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695907.705217:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695907.705221:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695907.705223:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695907.705231:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695907.705233:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695907.705236:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695907.705238:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139543296, portal 4 00000100:00000001:1.0:1787695907.705240:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695907.705241:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150912 : -128555779400704 : ffff8b14466f5c00) 00000100:00000040:1.0:1787695907.705245:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5c00 x1874531139543296/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.705250:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695907.705252:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695907.705253:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695907.705255:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139543296, offset 0 00000400:00000200:1.0:1787695907.705258:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695907.705261:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695907.705263:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695907.705265:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695907.705267:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695907.705269:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695907.705271:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695907.705273:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17054:17054:256:4294967295] 192.168.201.140@tcp LPNI seq info [17053:17053:8:4294967295] 00000400:00000200:1.0:1787695907.705276:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695907.705280:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695907.705283:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695907.705285:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695907.705288:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695907.705290:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1787695907.705293:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695907.705295:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1787695907.705297:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695907.705299:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695907.705301:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695907.705303:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695907.705305:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695907.705310:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695907.705312:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695907.705315:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5c00 x1874531139543296/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.705320:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695907.705322:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5c00 x1874531139543296/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695932 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.705327:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.705328:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695907.705331:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695907.705333:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695907.705335:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695907.705337:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298468197 00000100:00000001:1.0:1787695907.705338:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298479417 00000100:00000001:1.0:1787695907.705340:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298468197 00000100:00000001:1.0:1787695907.705342:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298479417 00000100:00000001:1.0:1787695907.705344:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695907.705346:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.705348:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695907.705352:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695907/real 1787695907] req@ffff8b14466f5c00 x1874531139543296/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695907.705360:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695907.705362:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695907.705382:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695907.705384:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695907.705385:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695907.705388:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695907.705393:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5c00 x1874531139543296/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695907.705403:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5c00 x1874531139543296/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.705419:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695907.705421:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.705423:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695907.705425:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695907.705430:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5c00 x1874531139543296/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.705437:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695907.705441:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5c00 x1874531139543296/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.705449:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695907.705452:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.705453:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695907.705455:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.705457:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695907.705458:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695907.705461:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695907.705464:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695907.705466:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695907.705470:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5c00 x1874531139543296/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695907.705478:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139543296:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695907.705482:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695907.705485:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6a00 x1874531139543424/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.705492:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695907.705494:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695907.705501:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695907.705503:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695907.705505:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695907.705508:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139543424:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695907.705512:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695907.705514:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695907.705515:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695907.705517:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695907.705520:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695907.705522:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695907.705526:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695907.705529:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695907.705533:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695907.705536:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139543424, portal 4 00000100:00000001:1.0:1787695907.705538:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695907.705540:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930154496 : -128555779397120 : ffff8b14466f6a00) 00000100:00000040:1.0:1787695907.705546:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6a00 x1874531139543424/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.705552:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695907.705555:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695907.705557:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695907.705559:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139543424, offset 0 00000400:00000200:1.0:1787695907.705563:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695907.705568:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695907.705571:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695907.705574:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695907.705577:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695907.705579:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695907.705582:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695907.705585:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17055:17055:256:4294967295] 192.168.201.140@tcp LPNI seq info [17054:17054:8:4294967295] 00000400:00000200:1.0:1787695907.705589:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695907.705596:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695907.705605:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695907.705608:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695907.705613:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695907.705615:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1787695907.705620:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695907.705623:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1787695907.705627:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695907.705628:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695907.705632:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695907.705635:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695907.705639:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695907.705641:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695907.705643:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695907.705647:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6a00 x1874531139543424/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.705656:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695907.705660:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6a00 x1874531139543424/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695932 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.705666:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.705668:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695907.705671:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695907.705674:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695907.705678:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695907.705680:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298467870 00000100:00000001:1.0:1787695907.705683:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298479417 00000100:00000001:1.0:1787695907.705686:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298467870 00000100:00000001:1.0:1787695907.705687:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298479417 00000100:00000001:1.0:1787695907.705690:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695907.705692:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.705694:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695907.705699:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695907/real 1787695907] req@ffff8b14466f6a00 x1874531139543424/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695907.705715:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695907.705716:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695907.705718:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695907.705721:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695907.705723:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695907.705725:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695907.705729:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6a00 x1874531139543424/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695907.705738:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6a00 x1874531139543424/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.705745:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695907.705747:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.705749:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695907.705750:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695907.705754:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6a00 x1874531139543424/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.705761:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695907.705765:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6a00 x1874531139543424/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.705772:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695907.705775:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.705776:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695907.705778:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.705780:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695907.705781:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695907.705784:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695907.705787:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695907.705790:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695907.705799:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6a00 x1874531139543424/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695907.705808:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139543424:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695907.705812:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695907.705815:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7b80 x1874531139543552/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.705821:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695907.705823:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695907.705826:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695907.705828:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695907.705830:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695907.705833:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139543552:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695907.705836:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695907.705838:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695907.705840:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695907.705842:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695907.705846:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695907.705848:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695907.705852:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695907.705854:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695907.705858:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695907.705861:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139543552, portal 10 00000100:00000001:1.0:1787695907.705863:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695907.705865:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930158976 : -128555779392640 : ffff8b14466f7b80) 00000100:00000040:1.0:1787695907.705872:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7b80 x1874531139543552/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.705879:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695907.705882:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695907.705884:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695907.705886:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139543552, offset 0 00000400:00000200:1.0:1787695907.705896:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695907.705901:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695907.705904:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695907.705908:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695907.705911:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695907.705913:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695907.705916:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695907.705919:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17056:17056:256:4294967295] 192.168.201.140@tcp LPNI seq info [17055:17055:8:4294967295] 00000400:00000200:1.0:1787695907.705924:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695907.705930:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695907.705935:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695907.705939:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695907.705943:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695907.705946:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1787695907.705951:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695907.705954:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1787695907.705963:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695907.705965:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695907.705969:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695907.705972:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695907.705976:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695907.705978:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695907.705981:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695907.705984:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7b80 x1874531139543552/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.705992:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695907.705996:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7b80 x1874531139543552/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695932 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.706002:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.706004:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695907.706007:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695907.706011:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695907.706028:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695907.706037:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298468056 00000100:00000001:1.0:1787695907.706040:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298479417 00000100:00000001:1.0:1787695907.706042:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298468056 00000100:00000001:1.0:1787695907.706044:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298479417 00000100:00000001:1.0:1787695907.706047:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695907.706049:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.706051:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695907.706056:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695907/real 1787695907] req@ffff8b14466f7b80 x1874531139543552/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695907.706065:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695907.706066:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695907.706068:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695907.706070:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695907.706072:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695907.706075:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695907.706079:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7b80 x1874531139543552/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695907.706087:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7b80 x1874531139543552/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.706094:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695907.706096:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.706098:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695907.706099:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695907.706103:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7b80 x1874531139543552/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.706111:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695907.706115:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7b80 x1874531139543552/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.706123:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695907.706130:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.706132:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695907.706134:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.706136:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695907.706137:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695907.706140:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695907.706144:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695907.706147:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695907.706151:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7b80 x1874531139543552/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695907.706160:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139543552:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695907.706165:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695907.706168:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7800 x1874531139543680/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.706175:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695907.706176:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695907.706179:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695907.706181:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695907.706183:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695907.706186:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139543680:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695907.706189:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695907.706191:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695907.706192:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695907.706194:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695907.706198:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695907.706201:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695907.706204:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695907.706207:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695907.706210:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695907.706214:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139543680, portal 10 00000100:00000001:1.0:1787695907.706221:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695907.706222:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930158080 : -128555779393536 : ffff8b14466f7800) 00000100:00000040:1.0:1787695907.706229:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7800 x1874531139543680/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.706236:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695907.706238:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695907.706240:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695907.706242:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139543680, offset 0 00000400:00000200:1.0:1787695907.706245:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695907.706248:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695907.706250:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695907.706252:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695907.706254:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695907.706256:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695907.706258:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695907.706260:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17057:17057:256:4294967295] 192.168.201.140@tcp LPNI seq info [17056:17056:8:4294967295] 00000400:00000200:1.0:1787695907.706263:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695907.706270:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695907.706274:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695907.706277:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695907.706281:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695907.706284:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1787695907.706288:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695907.706291:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1787695907.706295:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695907.706297:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695907.706300:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695907.706304:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695907.706307:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695907.706309:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695907.706312:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695907.706316:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7800 x1874531139543680/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695932 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.706333:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695907.706336:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7800 x1874531139543680/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695932 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.706340:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.706342:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695907.706345:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695907.706347:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695907.706349:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695907.706351:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298468034 00000100:00000001:1.0:1787695907.706353:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298479417 00000100:00000001:1.0:1787695907.706354:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298468034 00000100:00000001:1.0:1787695907.706356:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298479417 00000100:00000001:1.0:1787695907.706357:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695907.706358:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.706359:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695907.706362:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695907/real 1787695907] req@ffff8b14466f7800 x1874531139543680/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695907.706383:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695907.706384:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695907.706386:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695907.706389:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695907.706390:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695907.706393:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695907.706396:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7800 x1874531139543680/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695907.706405:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7800 x1874531139543680/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.706412:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695907.706419:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.706421:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695907.706422:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695907.706427:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7800 x1874531139543680/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.706434:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695907.706439:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7800 x1874531139543680/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.706446:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695907.706449:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.706451:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695907.706452:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.706453:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695907.706454:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695907.706456:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695907.706458:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695907.706460:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695907.706463:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7800 x1874531139543680/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695907.706468:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139543680:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695907.706470:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.706472:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695907.706474:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5c00 x1874531139543296/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.706477:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695907.706479:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695907.706480:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695907.706483:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695907.706484:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695907.706489:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695907.706490:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695907.706492:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676000. 02000000:00000001:1.0:1787695907.706495:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695907.706496:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695907.706497:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5c00. 00000100:00000001:1.0:1787695907.706499:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695907.706500:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.706501:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695907.706503:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6a00 x1874531139543424/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.706508:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695907.706509:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695907.706510:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695907.706511:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695907.706512:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695907.706512:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695907.706514:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695907.706514:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:1.0:1787695907.706516:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695907.706517:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695907.706517:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6a00. 00000100:00000001:1.0:1787695907.706519:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695907.706519:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.706520:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695907.706522:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7b80 x1874531139543552/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.706526:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695907.706527:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695907.706527:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695907.706529:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695907.706529:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695907.706530:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695907.706532:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695907.706533:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:1.0:1787695907.706534:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695907.706537:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695907.706538:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7b80. 00000100:00000001:1.0:1787695907.706540:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695907.706541:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.706542:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695907.706544:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7800 x1874531139543680/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695932 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695907.706548:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695907.706549:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695907.706550:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695907.706552:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695907.706553:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695907.706554:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695907.706555:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695907.706556:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:1.0:1787695907.706557:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695907.706558:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695907.706559:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7800. 00000100:00000001:1.0:1787695907.706560:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695907.706561:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.706562:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695907.706564:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695907.706565:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695907.706566:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695907.706567:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695907.766461:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695907.766476:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695907.766479:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695907.766484:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695907.766488:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695907.766492:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695907.766495:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695907.766498:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695907.766501:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695908.406499:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695908.406505:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695908.406512:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695908.406513:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695908.406516:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695908.406518:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695908.406521:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695908.406522:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695908.406525:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695908.406527:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695908.406530:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695908.406531:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695908.406608:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695908.406611:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695908.406614:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695908.790416:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695908.790422:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695908.790424:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695908.790428:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695908.790431:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695908.790433:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695908.790435:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695908.790437:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695908.790439:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695909.430500:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695909.430508:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695909.430515:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695909.430516:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695909.430520:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695909.430522:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695909.430526:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695909.430527:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695909.430531:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695909.430533:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695909.430536:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695909.430537:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695909.430588:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695909.430591:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695909.430594:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695909.814470:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695909.814477:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695909.814478:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695909.814490:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695909.814494:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695909.814499:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695909.814501:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695909.814504:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695909.814507:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695910.454458:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695910.454464:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695910.454468:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695910.454469:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695910.454472:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695910.454473:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695910.454476:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695910.454477:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695910.454481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695910.454482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695910.454485:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695910.454487:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695910.454539:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695910.454543:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695910.454547:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695910.838476:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695910.838484:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695910.838485:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695910.838489:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695910.838509:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695910.838511:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695910.838513:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695910.838515:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695910.838517:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695911.478498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695911.478503:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695911.478513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695911.478514:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695911.478517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695911.478518:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695911.478520:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695911.478521:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695911.478523:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695911.478526:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695911.478528:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695911.478533:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695911.478534:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695911.478535:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695911.478554:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695911.478556:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695911.478558:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695911.478559:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695911.478562:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695911.478563:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695911.478624:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695911.478629:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695911.478633:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695911.862449:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695911.862455:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695911.862456:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695911.862461:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695911.862463:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695911.862467:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695911.862468:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695911.862471:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695911.862473:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695912.502471:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695912.502479:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695912.502486:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695912.502487:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695912.502491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695912.502492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695912.502496:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695912.502497:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695912.502501:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695912.502502:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695912.502506:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695912.502507:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695912.502559:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695912.502564:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695912.502567:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787695912.822766:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695912.822778:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787695912.822780:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695912.822784:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695912.822788:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695912.822795:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3800 00000400:00000200:3.0:1787695912.822804:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695912.822810:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695912.822813:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695912.822817:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.822820:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695912.822823:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.822825:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695912.822829:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695912.822831:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:3.0:1787695912.822834:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695912.822837:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787695912.822848:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787695912.822850:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787695912.822854:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.822857:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695912.822859:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.822861:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695912.822864:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787695912.822867:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695912.822870:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695912.822873:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695912.822876:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695912.822879:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695912.822881:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787695912.822884:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695912.822890:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7480. 00000020:00000040:3.0:1787695912.822894:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787695912.822898:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695912.822900:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695912.822903:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695912.822906:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695912.822910:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695912.822916:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674800. 00000100:00000001:3.0:1787695912.822923:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695912.822945:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7480 x1874531139543808/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695912.822957:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695912.822961:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695912.822966:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043daeda0] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7480 x1874531139543808/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695912.822991:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695912.822995:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695912.823003:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695912.823005:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695912.823007:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695912.823010:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695912.823013:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3800 00000400:00000200:3.0:1787695912.823017:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695912.823020:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695912.823022:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695912.823025:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823027:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695912.823029:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823031:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695912.823033:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695912.823034:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:3.0:1787695912.823036:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695912.823038:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:3.0:1787695912.823040:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:3.0:1787695912.823042:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:3.0:1787695912.823045:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823046:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695912.823048:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823050:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695912.823052:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:3.0:1787695912.823055:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823057:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695912.823059:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695912.823064:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695912.823067:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695912.823069:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695912.823074:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695912.823081:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695912.823083:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695912.823087:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4e00. 00000020:00000040:3.0:1787695912.823089:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695912.823093:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695912.823094:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695912.823096:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695912.823097:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695912.823099:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695912.823105:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 00000100:00000001:3.0:1787695912.823109:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695912.823114:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4e00 x1874531139543936/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695912.823122:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695912.823124:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695912.823128:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcd77cd6] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4e00 x1874531139543936/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695912.823135:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695912.823138:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695912.823141:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695912.823142:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695912.823144:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695912.823147:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695912.823149:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3800 00000400:00000200:3.0:1787695912.823153:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695912.823157:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695912.823159:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695912.823162:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823164:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695912.823166:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823173:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695912.823175:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695912.823176:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:3.0:1787695912.823178:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695912.823180:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:3.0:1787695912.823182:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:3.0:1787695912.823184:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:3.0:1787695912.823187:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823189:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695912.823191:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823193:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695912.823195:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:3.0:1787695912.823198:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823200:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695912.823201:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695912.823204:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695912.823207:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695912.823209:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695912.823212:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695912.823214:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695912.823215:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695912.823218:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7100. 00000020:00000040:3.0:1787695912.823220:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695912.823223:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695912.823224:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695912.823226:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695912.823227:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695912.823229:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695912.823232:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 00000100:00000001:3.0:1787695912.823235:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695912.823239:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7100 x1874531139544064/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695912.823256:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695912.823257:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695912.823261:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000004e8abd7] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7100 x1874531139544064/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695912.823268:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695912.823271:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695912.823273:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787695912.823275:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695912.823276:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695912.823278:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695912.823281:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3800 00000400:00000200:3.0:1787695912.823284:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695912.823288:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695912.823290:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695912.823292:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823294:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695912.823296:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823298:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695912.823300:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695912.823301:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:3.0:1787695912.823303:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695912.823305:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:3.0:1787695912.823306:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:3.0:1787695912.823308:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:3.0:1787695912.823311:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823313:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695912.823315:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823317:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695912.823326:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:3.0:1787695912.823329:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823332:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695912.823334:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695912.823337:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695912.823339:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695912.823340:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695912.823343:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787695912.823345:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695912.823347:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695912.823351:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7800. 00000020:00000040:3.0:1787695912.823353:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:3.0:1787695912.823355:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695912.823357:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695912.823358:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695912.823360:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695912.823362:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695912.823390:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 00000100:00000001:3.0:1787695912.823394:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695912.823400:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7800 x1874531139544192/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695912.823408:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695912.823411:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695912.823415:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7c57981] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7800 x1874531139544192/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695912.823424:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695912.823426:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695912.823430:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1787695912.823432:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695912.823433:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695912.823435:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695912.823443:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3800 00000400:00000200:3.0:1787695912.823448:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695912.823451:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695912.823453:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695912.823462:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823464:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695912.823466:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823468:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695912.823471:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695912.823473:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:3.0:1787695912.823475:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695912.823477:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:3.0:1787695912.823479:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:3.0:1787695912.823482:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:3.0:1787695912.823485:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695912.823486:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695912.823488:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695912.823491:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695912.823492:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695912.823494:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:2.0:1787695912.823497:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1787695912.823498:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:3.0:1787695912.823503:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823506:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695912.823508:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:2.0:1787695912.823509:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7480 x1874531139543808/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695912.823512:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695912.823517:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695912.823522:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787695912.823525:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1787695912.823526:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787695912.823527:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787695912.823532:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695912.823533:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695912.823537:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695912.823539:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695912.823542:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139543808:192.168.201.140@tcp:250:kworker.0 00000100:00000001:3.0:1787695912.823545:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695912.823546:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695912.823551:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:3.0:1787695912.823552:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7b80. 02000000:00000001:2.0:1787695912.823554:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695912.823556:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:3.0:1787695912.823561:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000010:2.0:1787695912.823565:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445662000. 02000000:00000001:3.0:1787695912.823565:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695912.823569:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695912.823573:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695912.823578:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695912.823579:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695912.823582:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:2.0:1787695912.823585:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 02000000:00000001:3.0:1787695912.823585:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695912.823593:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676000. 00000400:00000010:2.0:1787695912.823596:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000001:3.0:1787695912.823597:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1787695912.823605:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139543808, portal 25 00000100:00100000:3.0:1787695912.823606:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7b80 x1874531139544320/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695912.823612:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695912.823616:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930157184 : -128555779394432 : ffff8b14466f7480) 00000100:00000001:3.0:1787695912.823617:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695912.823621:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695912.823625:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7480 x1874531139543808/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695912.823629:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b87e1da] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7b80 x1874531139544320/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.823644:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787695912.823648:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695912.823652:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00080000:3.0:1787695912.823654:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:2.0:1787695912.823657:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00080000:3.0:1787695912.823658:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695912.823662:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:2.0:1787695912.823664:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139543808, offset 0 00000100:00080000:3.0:1787695912.823665:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695912.823671:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695912.823676:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3800 00000400:00000200:2.0:1787695912.823677:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:3.0:1787695912.823682:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695912.823686:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695912.823688:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1787695912.823690:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695912.823695:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:2.0:1787695912.823699:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:3.0:1787695912.823701:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695912.823704:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823708:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695912.823713:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695912.823714:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000400:00000200:2.0:1787695912.823715:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1787695912.823718:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695912.823722:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000400:00000200:2.0:1787695912.823724:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000020:00000001:3.0:1787695912.823727:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000200:2.0:1787695912.823730:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000020:00000001:3.0:1787695912.823738:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000040:3.0:1787695912.823743:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823745:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695912.823747:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823752:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695912.823754:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000400:00000200:2.0:1787695912.823757:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00080000:3.0:1787695912.823761:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823767:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695912.823768:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17058:17058:256:4294967295] 192.168.201.140@tcp LPNI seq info [17057:17057:8:4294967295] 02000000:00000001:3.0:1787695912.823771:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695912.823776:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1787695912.823779:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787695912.823781:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695912.823785:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695912.823788:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695912.823791:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695912.823795:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695912.823800:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000010:3.0:1787695912.823801:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6a00. 00000020:00000040:3.0:1787695912.823805:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:3.0:1787695912.823807:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695912.823809:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695912.823811:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695912.823814:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695912.823817:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695912.823821:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 00000800:00000200:2.0:1787695912.823826:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695912.823827:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1787695912.823833:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000100:00100000:3.0:1787695912.823835:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6a00 x1874531139544448/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:2.0:1787695912.823847:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787695912.823849:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695912.823853:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1787695912.823854:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000100:00000040:3.0:1787695912.823859:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002ad47e8f] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6a00 x1874531139544448/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:2.1F:1787695912.823867:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787695912.823872:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1787695912.823878:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000100:00080000:3.0:1787695912.823883:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787695912.823889:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00080000:3.0:1787695912.823891:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000010:2.0:1787695912.823893:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000100:00000001:3.0:1787695912.823894:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695912.823898:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1787695912.823901:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1787695912.823903:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695912.823909:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3800 00000400:00000200:2.0:1787695912.823911:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:3.0:1787695912.823917:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695912.823921:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000200:2.0:1787695912.823943:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695912.823947:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695912.823952:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695912.823958:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7480 x1874531139543808/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.823970:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695912.823974:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7480 x1874531139543808/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695937 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.823984:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695912.823986:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695912.823992:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695912.823997:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695912.824001:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298484537 00000100:00000001:2.0:1787695912.824008:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298479417 00000100:00000001:2.0:1787695912.824013:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298484537 00000100:00000001:2.0:1787695912.824024:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298479417 00000100:00000001:2.0:1787695912.824027:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695912.824030:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695912.824033:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695912.824038:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695912/real 1787695912] req@ffff8b14466f7480 x1874531139543808/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695912.824050:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695912.824052:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695912.824054:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695912.824057:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695912.824059:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695912.824062:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695912.824066:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7480 x1874531139543808/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695912.824075:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7480 x1874531139543808/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.824082:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695912.824084:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.824086:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695912.824089:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695912.824093:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7480 x1874531139543808/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.824100:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695912.824105:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7480 x1874531139543808/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.824113:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695912.824116:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.824118:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695912.824119:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.824122:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695912.824130:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695912.824133:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695912.824137:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695912.824139:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695912.824144:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7480 x1874531139543808/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695912.824153:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139543808:192.168.201.140@tcp:250:kworker.0 00000100:00000001:2.0:1787695912.824158:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695912.824161:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4e00 x1874531139543936/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.824168:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695912.824170:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695912.824172:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695912.824174:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695912.824177:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695912.824180:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139543936:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695912.824183:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695912.824185:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695912.824187:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695912.824189:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695912.824192:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445662800. 02000000:00000001:2.0:1787695912.824195:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695912.824199:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695912.824202:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695912.824206:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695912.824209:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139543936, portal 10 00000100:00000001:2.0:1787695912.824211:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695912.824213:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930147328 : -128555779404288 : ffff8b14466f4e00) 00000100:00000040:2.0:1787695912.824220:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4e00 x1874531139543936/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.824239:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695912.824242:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695912.824245:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695912.824248:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139543936, offset 0 00000400:00000200:2.0:1787695912.824252:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695912.824258:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:3.0:1787695912.824260:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1787695912.824265:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000040:3.0:1787695912.824268:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:2.0:1787695912.824271:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1787695912.824273:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1787695912.824279:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000040:3.0:1787695912.824279:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.824285:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:2.0:1787695912.824287:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000020:00000001:3.0:1787695912.824289:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695912.824292:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000400:00000200:2.0:1787695912.824295:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000020:00000001:3.0:1787695912.824297:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695912.824301:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000400:00000200:2.0:1787695912.824302:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17059:17059:256:4294967295] 192.168.201.140@tcp LPNI seq info [17058:17058:8:4294967295] 00000020:00000001:3.0:1787695912.824305:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000020:00000001:3.0:1787695912.824311:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000400:00000200:2.0:1787695912.824316:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695912.824319:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.824323:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695912.824330:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695912.824334:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:2.0:1787695912.824335:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000020:00000040:3.0:1787695912.824339:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000800:00000200:2.0:1787695912.824346:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00080000:3.0:1787695912.824346:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695912.824353:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1787695912.824354:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 02000000:00000001:3.0:1787695912.824355:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695912.824362:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695912.824389:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695912.824391:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695912.824400:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000800:00000200:2.0:1787695912.824402:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000008:00000001:3.0:1787695912.824405:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1787695912.824410:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000100:00000001:3.0:1787695912.824410:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695912.824414:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5c00. 00000800:00000200:2.1:1787695912.824420:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000020:00000040:3.0:1787695912.824421:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000800:00000200:2.1:1787695912.824427:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 02000000:00000001:3.0:1787695912.824428:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695912.824431:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695912.824435:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000100:2.0:1787695912.824436:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695912.824437:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000010:2.0:1787695912.824441:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 02000000:00000001:3.0:1787695912.824442:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695912.824449:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000010:3.0:1787695912.824450:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675400. 00000400:00000200:2.0:1787695912.824457:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1787695912.824457:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695912.824464:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000100:00100000:3.0:1787695912.824468:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5c00 x1874531139544576/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1787695912.824469:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695912.824475:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787695912.824483:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000200:2.0:1787695912.824484:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4e00 x1874531139543936/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695912.824488:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695912.824498:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005bfbcee1] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5c00 x1874531139544576/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.824501:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695912.824509:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4e00 x1874531139543936/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695937 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695912.824511:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695912.824516:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787695912.824525:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695912.824529:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787695912.824534:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695912.824543:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3810) 00000100:00000001:2.0:1787695912.824545:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695912.824549:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695912.824553:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695912.824557:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000400:00000200:2.0:1787695912.824562:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695912.824564:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298474439 00000100:00000001:2.0:1787695912.824567:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298484537 00000100:00000001:2.0:1787695912.824569:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298474439 00000100:00000001:2.0:1787695912.824572:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298484537 00000100:00000001:2.0:1787695912.824574:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695912.824576:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695912.824578:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695912.824583:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695912/real 1787695912] req@ffff8b14466f4e00 x1874531139543936/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695912.824591:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695912.824593:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695912.824595:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695912.824598:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695912.824599:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695912.824602:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695912.824605:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4e00 x1874531139543936/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695912.824614:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4e00 x1874531139543936/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.824620:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695912.824622:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.824624:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695912.824626:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695912.824630:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4e00 x1874531139543936/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.824645:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695912.824650:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4e00 x1874531139543936/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.824657:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695912.824660:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.824662:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695912.824663:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.824665:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695912.824667:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695912.824669:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695912.824672:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695912.824674:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695912.824678:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4e00 x1874531139543936/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695912.824685:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139543936:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695912.824689:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695912.824692:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7100 x1874531139544064/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.824699:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695912.824700:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695912.824703:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695912.824705:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695912.824707:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695912.824710:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139544064:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695912.824714:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695912.824717:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695912.824723:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695912.824725:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695912.824728:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445661c00. 02000000:00000001:2.0:1787695912.824730:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695912.824734:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695912.824737:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695912.824740:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695912.824744:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139544064, portal 10 00000100:00000001:2.0:1787695912.824746:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695912.824747:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930156288 : -128555779395328 : ffff8b14466f7100) 00000100:00000040:2.0:1787695912.824753:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7100 x1874531139544064/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.824759:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695912.824762:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695912.824764:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695912.824766:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139544064, offset 0 00000400:00000200:2.0:1787695912.824770:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695912.824775:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695912.824778:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695912.824781:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695912.824783:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695912.824786:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695912.824788:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695912.824791:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17060:17060:256:4294967295] 192.168.201.140@tcp LPNI seq info [17059:17059:8:4294967295] 00000400:00000200:2.0:1787695912.824795:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695912.824802:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695912.824806:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695912.824809:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695912.824813:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695912.824816:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1787695912.824821:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695912.824824:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1787695912.824828:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695912.824835:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695912.824839:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695912.824843:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695912.824846:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695912.824849:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695912.824851:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695912.824855:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7100 x1874531139544064/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.824864:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695912.824867:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7100 x1874531139544064/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695937 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.824873:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695912.824875:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695912.824878:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695912.824881:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:2.0:1787695912.824884:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695912.824886:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298473287 00000100:00000001:2.0:1787695912.824889:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298484537 00000100:00000001:2.0:1787695912.824891:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298473287 00000100:00000001:2.0:1787695912.824893:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298484537 00000100:00000001:2.0:1787695912.824895:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695912.824897:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695912.824899:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695912.824903:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695912/real 1787695912] req@ffff8b14466f7100 x1874531139544064/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695912.824911:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695912.824912:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695912.824914:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695912.824917:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695912.824918:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695912.824941:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695912.824945:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7100 x1874531139544064/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695912.824954:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7100 x1874531139544064/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.824961:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695912.824962:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.824964:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695912.824966:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695912.824970:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7100 x1874531139544064/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.824977:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695912.824981:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7100 x1874531139544064/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.824988:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695912.824991:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.824993:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695912.824994:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.824996:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695912.824998:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695912.825000:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695912.825004:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695912.825006:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695912.825010:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7100 x1874531139544064/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695912.825018:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139544064:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695912.825022:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695912.825032:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7800 x1874531139544192/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.825038:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695912.825040:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695912.825042:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695912.825044:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695912.825046:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695912.825049:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139544192:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787695912.825052:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695912.825054:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695912.825055:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695912.825057:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695912.825060:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445660800. 02000000:00000001:2.0:1787695912.825062:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695912.825066:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695912.825068:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695912.825072:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695912.825076:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139544192, portal 4 00000100:00000001:2.0:1787695912.825078:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695912.825079:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930158080 : -128555779393536 : ffff8b14466f7800) 00000100:00000040:2.0:1787695912.825086:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7800 x1874531139544192/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.825092:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695912.825094:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695912.825097:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695912.825099:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139544192, offset 0 00000400:00000200:2.0:1787695912.825103:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695912.825107:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695912.825112:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695912.825114:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695912.825117:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695912.825119:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695912.825122:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695912.825130:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17061:17061:256:4294967295] 192.168.201.140@tcp LPNI seq info [17060:17060:8:4294967295] 00000400:00000200:2.0:1787695912.825134:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695912.825141:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695912.825146:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695912.825149:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695912.825153:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695912.825156:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1787695912.825161:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695912.825164:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1787695912.825168:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695912.825170:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695912.825173:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695912.825176:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695912.825179:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695912.825181:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695912.825184:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695912.825188:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7800 x1874531139544192/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.825195:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695912.825199:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7800 x1874531139544192/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695937 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.825205:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695912.825207:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695912.825210:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695912.825213:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:2.0:1787695912.825216:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695912.825218:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298473317 00000100:00000001:2.0:1787695912.825221:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298484537 00000100:00000001:2.0:1787695912.825223:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298473317 00000100:00000001:2.0:1787695912.825225:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298484537 00000100:00000001:2.0:1787695912.825228:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695912.825234:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695912.825236:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695912.825240:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695912/real 1787695912] req@ffff8b14466f7800 x1874531139544192/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695912.825249:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695912.825250:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695912.825252:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695912.825254:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695912.825256:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695912.825258:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695912.825261:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7800 x1874531139544192/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695912.825270:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7800 x1874531139544192/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.825277:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695912.825278:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.825280:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695912.825282:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695912.825285:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7800 x1874531139544192/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.825293:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695912.825297:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7800 x1874531139544192/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.825304:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695912.825307:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.825308:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695912.825310:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.825312:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695912.825313:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695912.825315:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695912.825324:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695912.825327:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695912.825331:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7800 x1874531139544192/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695912.825339:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139544192:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787695912.825343:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.825346:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695912.825349:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7480 x1874531139543808/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.825355:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695912.825358:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695912.825360:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445662000. 02000000:00000001:2.0:1787695912.825382:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695912.825384:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695912.825386:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:2.0:1787695912.825388:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695912.825391:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674800. 02000000:00000001:2.0:1787695912.825396:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695912.825398:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695912.825400:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7480. 00000100:00000001:2.0:1787695912.825404:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695912.825406:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.825408:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695912.825413:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4e00 x1874531139543936/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.825420:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695912.825421:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695912.825423:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445662800. 02000000:00000001:2.0:1787695912.825425:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695912.825427:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695912.825428:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:2.0:1787695912.825430:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695912.825436:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:2.0:1787695912.825439:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695912.825440:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695912.825442:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4e00. 00000100:00000001:2.0:1787695912.825445:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695912.825446:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.825448:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695912.825452:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7100 x1874531139544064/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.825459:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695912.825460:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695912.825462:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445661c00. 02000000:00000001:2.0:1787695912.825464:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695912.825466:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695912.825468:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:2.0:1787695912.825470:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695912.825472:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:2.0:1787695912.825475:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695912.825477:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695912.825478:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7100. 00000100:00000001:2.0:1787695912.825480:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695912.825482:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.825484:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695912.825487:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7800 x1874531139544192/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.825493:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695912.825494:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695912.825496:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445660800. 02000000:00000001:2.0:1787695912.825498:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695912.825499:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695912.825500:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:2.0:1787695912.825503:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695912.825504:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:2.0:1787695912.825508:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695912.825509:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695912.825511:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7800. 00000100:00000001:2.0:1787695912.825518:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695912.825519:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.825522:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.825528:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695912.825531:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695912.825533:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695912.825535:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695912.825537:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695912.825541:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7b80 x1874531139544320/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.825549:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695912.825550:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695912.825552:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695912.825554:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695912.825557:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695912.825559:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139544320:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787695912.825563:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695912.825565:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695912.825566:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695912.825568:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695912.825571:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445660800. 02000000:00000001:2.0:1787695912.825573:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695912.825577:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695912.825579:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695912.825583:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695912.825586:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139544320, portal 4 00000100:00000001:2.0:1787695912.825588:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695912.825589:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930158976 : -128555779392640 : ffff8b14466f7b80) 00000100:00000040:2.0:1787695912.825595:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7b80 x1874531139544320/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.825601:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695912.825604:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695912.825606:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695912.825608:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139544320, offset 0 00000400:00000200:2.0:1787695912.825617:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695912.825621:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695912.825624:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695912.825627:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695912.825629:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695912.825632:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695912.825634:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695912.825638:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17062:17062:256:4294967295] 192.168.201.140@tcp LPNI seq info [17061:17061:8:4294967295] 00000400:00000200:2.0:1787695912.825642:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695912.825648:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695912.825653:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695912.825655:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695912.825660:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695912.825664:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1787695912.825669:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695912.825672:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1787695912.825677:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695912.825679:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695912.825682:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695912.825687:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695912.825690:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695912.825692:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695912.825695:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695912.825699:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7b80 x1874531139544320/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.825707:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695912.825710:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7b80 x1874531139544320/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695937 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.825717:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695912.825719:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695912.825721:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695912.825724:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:2.0:1787695912.825728:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695912.825734:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298472990 00000100:00000001:2.0:1787695912.825737:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298484537 00000100:00000001:2.0:1787695912.825739:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298472990 00000100:00000001:2.0:1787695912.825742:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298484537 00000100:00000001:2.0:1787695912.825744:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695912.825746:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695912.825748:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695912.825752:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695912/real 1787695912] req@ffff8b14466f7b80 x1874531139544320/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695912.825760:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695912.825762:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695912.825764:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695912.825766:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695912.825768:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695912.825770:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695912.825773:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7b80 x1874531139544320/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695912.825782:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7b80 x1874531139544320/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.825789:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695912.825791:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.825792:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695912.825794:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695912.825797:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7b80 x1874531139544320/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.825805:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695912.825809:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7b80 x1874531139544320/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.825815:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695912.825822:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.825824:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695912.825825:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.825827:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695912.825829:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695912.825832:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695912.825835:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695912.825837:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695912.825841:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7b80 x1874531139544320/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695912.825851:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139544320:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787695912.825856:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695912.825859:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6a00 x1874531139544448/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.825866:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695912.825868:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695912.825870:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695912.825872:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695912.825874:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695912.825877:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139544448:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695912.825880:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695912.825882:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695912.825884:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695912.825886:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695912.825888:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445661c00. 02000000:00000001:2.0:1787695912.825890:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695912.825893:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695912.825896:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695912.825899:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695912.825902:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139544448, portal 10 00000100:00000001:2.0:1787695912.825908:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695912.825909:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930154496 : -128555779397120 : ffff8b14466f6a00) 00000100:00000040:2.0:1787695912.825916:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6a00 x1874531139544448/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.825922:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695912.825924:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695912.825941:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695912.825944:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139544448, offset 0 00000400:00000200:2.0:1787695912.825948:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695912.825952:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695912.825956:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695912.825959:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695912.825961:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695912.825963:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695912.825966:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695912.825969:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17063:17063:256:4294967295] 192.168.201.140@tcp LPNI seq info [17062:17062:8:4294967295] 00000400:00000200:2.0:1787695912.825973:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695912.825978:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695912.825983:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695912.825986:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695912.825990:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695912.825993:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1787695912.825996:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695912.825999:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1787695912.826002:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695912.826005:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695912.826008:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695912.826011:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695912.826014:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695912.826015:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695912.826018:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695912.826021:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6a00 x1874531139544448/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.826041:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695912.826044:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6a00 x1874531139544448/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695937 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.826051:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695912.826052:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695912.826055:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695912.826058:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:2.0:1787695912.826061:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695912.826063:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298473176 00000100:00000001:2.0:1787695912.826066:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298484537 00000100:00000001:2.0:1787695912.826068:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298473176 00000100:00000001:2.0:1787695912.826070:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298484537 00000100:00000001:2.0:1787695912.826072:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695912.826074:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695912.826075:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695912.826079:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695912/real 1787695912] req@ffff8b14466f6a00 x1874531139544448/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695912.826086:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695912.826087:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695912.826089:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695912.826091:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695912.826093:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695912.826095:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695912.826098:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6a00 x1874531139544448/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695912.826106:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6a00 x1874531139544448/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.826111:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695912.826119:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.826121:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695912.826122:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695912.826126:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6a00 x1874531139544448/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.826132:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695912.826136:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6a00 x1874531139544448/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.826143:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695912.826145:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.826146:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695912.826148:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.826149:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695912.826151:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695912.826153:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695912.826156:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695912.826158:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695912.826161:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6a00 x1874531139544448/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695912.826168:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139544448:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695912.826171:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695912.826175:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5c00 x1874531139544576/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.826180:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695912.826182:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695912.826183:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695912.826185:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695912.826187:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695912.826194:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139544576:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695912.826197:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695912.826199:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695912.826200:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695912.826202:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695912.826205:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445662800. 02000000:00000001:2.0:1787695912.826207:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695912.826210:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695912.826212:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695912.826215:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695912.826218:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139544576, portal 10 00000100:00000001:2.0:1787695912.826220:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695912.826222:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150912 : -128555779400704 : ffff8b14466f5c00) 00000100:00000040:2.0:1787695912.826227:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5c00 x1874531139544576/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.826233:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695912.826235:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695912.826237:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695912.826239:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139544576, offset 0 00000400:00000200:2.0:1787695912.826243:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695912.826246:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695912.826250:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695912.826252:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695912.826255:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695912.826257:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695912.826259:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695912.826262:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17064:17064:256:4294967295] 192.168.201.140@tcp LPNI seq info [17063:17063:8:4294967295] 00000400:00000200:2.0:1787695912.826266:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695912.826272:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695912.826276:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695912.826279:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695912.826287:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695912.826289:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000200:2.1:1787695912.826293:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695912.826295:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000100:2.0:1787695912.826299:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695912.826301:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695912.826304:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695912.826307:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695912.826310:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695912.826312:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695912.826314:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695912.826318:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5c00 x1874531139544576/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695937 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.826325:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695912.826328:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5c00 x1874531139544576/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695937 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.826334:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695912.826336:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695912.826338:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695912.826341:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:2.0:1787695912.826344:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695912.826346:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298473154 00000100:00000001:2.0:1787695912.826349:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298484537 00000100:00000001:2.0:1787695912.826350:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298473154 00000100:00000001:2.0:1787695912.826353:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298484537 00000100:00000001:2.0:1787695912.826355:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695912.826357:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695912.826359:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695912.826379:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695912/real 1787695912] req@ffff8b14466f5c00 x1874531139544576/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695912.826388:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695912.826395:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695912.826397:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695912.826400:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695912.826401:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695912.826404:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695912.826408:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5c00 x1874531139544576/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695912.826415:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5c00 x1874531139544576/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.826421:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695912.826423:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.826424:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695912.826426:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695912.826429:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5c00 x1874531139544576/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.826435:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695912.826439:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5c00 x1874531139544576/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.826445:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695912.826447:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.826449:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695912.826450:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.826452:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695912.826453:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695912.826456:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695912.826458:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695912.826460:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695912.826463:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5c00 x1874531139544576/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695912.826482:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139544576:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695912.826486:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.826488:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695912.826491:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7b80 x1874531139544320/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.826498:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695912.826499:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695912.826501:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445660800. 02000000:00000001:2.0:1787695912.826503:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695912.826505:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695912.826507:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:2.0:1787695912.826509:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695912.826510:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676000. 02000000:00000001:2.0:1787695912.826513:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695912.826514:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695912.826515:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7b80. 00000100:00000001:2.0:1787695912.826518:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695912.826519:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.826521:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695912.826524:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6a00 x1874531139544448/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.826530:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695912.826531:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695912.826533:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445661c00. 02000000:00000001:2.0:1787695912.826535:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695912.826536:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695912.826537:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:2.0:1787695912.826539:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695912.826540:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:2.0:1787695912.826543:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695912.826544:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695912.826545:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6a00. 00000100:00000001:2.0:1787695912.826552:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695912.826553:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.826555:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695912.826558:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5c00 x1874531139544576/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695937 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695912.826564:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695912.826566:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695912.826567:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445662800. 02000000:00000001:2.0:1787695912.826569:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695912.826571:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695912.826572:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:2.0:1787695912.826574:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695912.826576:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675400. 02000000:00000001:2.0:1787695912.826579:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695912.826580:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695912.826581:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5c00. 00000100:00000001:2.0:1787695912.826583:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695912.826584:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.826586:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695912.826590:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695912.826592:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695912.826593:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695912.826595:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695912.886475:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695912.886482:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695912.886484:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695912.886488:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695912.886491:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695912.886500:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695912.886502:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695912.886518:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695912.886520:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695913.526435:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695913.526444:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695913.526446:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695913.526458:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695913.526463:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695913.526465:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695913.526467:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695913.526470:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695913.526473:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695913.526474:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695913.526476:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695913.526477:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695913.526478:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695913.526479:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695913.526480:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695913.526482:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695913.526484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695913.526485:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695913.526487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695913.526488:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695913.526511:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695913.526512:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695913.526515:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695913.526516:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695913.526575:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695913.526579:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695913.526584:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695913.910499:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695913.910507:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695913.910509:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695913.910516:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695913.910520:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695913.910524:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695913.910526:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695913.910529:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695913.910532:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695914.550494:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695914.550501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695914.550508:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695914.550509:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695914.550513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695914.550515:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695914.550518:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695914.550520:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695914.550523:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695914.550524:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695914.550527:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695914.550529:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695914.550587:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695914.550591:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695914.550594:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695914.934514:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695914.934523:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695914.934525:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695914.934532:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695914.934535:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695914.934539:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695914.934541:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695914.934543:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695914.934546:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695915.574475:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695915.574482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695915.574489:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695915.574490:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695915.574494:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695915.574495:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695915.574499:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695915.574501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695915.574504:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695915.574505:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695915.574509:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695915.574510:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695915.574583:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695915.574589:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695915.574594:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695915.958468:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695915.958475:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695915.958477:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695915.958483:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695915.958487:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695915.958491:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695915.958494:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695915.958496:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695915.958499:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695916.598451:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695916.598456:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695916.598461:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695916.598462:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695916.598464:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695916.598467:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695916.598470:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695916.598474:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695916.598476:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695916.598477:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695916.598495:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695916.598497:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695916.598498:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695916.598500:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695916.598503:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695916.598504:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695916.598505:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695916.598506:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695916.598507:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695916.598508:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695916.598509:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695916.598510:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695916.598511:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695916.598513:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695916.598515:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695916.598516:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695916.598519:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695916.598520:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695916.598521:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695916.598522:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695916.598522:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695916.598523:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695916.598524:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695916.598525:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695916.598526:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695916.598527:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695916.598587:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695916.598592:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695916.598595:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695916.982467:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695916.982474:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695916.982476:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695916.982481:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695916.982483:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695916.982487:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695916.982505:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695916.982507:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695916.982509:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695917.622483:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695917.622489:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695917.622494:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695917.622496:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695917.622500:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695917.622501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695917.622505:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695917.622507:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695917.622510:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695917.622511:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695917.622515:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695917.622516:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695917.622578:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695917.622581:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695917.622583:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787695917.942726:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695917.942744:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787695917.942747:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695917.942750:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695917.942754:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695917.942762:0:47074:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3805 00000400:00000200:1.0:1787695917.942771:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695917.942778:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695917.942781:0:47074:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695917.942785:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695917.942789:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695917.942792:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695917.942795:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695917.942799:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695917.942802:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:1.0:1787695917.942805:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695917.942809:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:1.0:1787695917.942817:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:1.0:1787695917.942820:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:1.0:1787695917.942824:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695917.942826:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695917.942828:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695917.942830:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695917.942833:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:1.0:1787695917.942852:0:47074:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695917.942855:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695917.942858:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695917.942862:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695917.942865:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695917.942867:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695917.942870:0:47074:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695917.942876:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665f80. 00000020:00000040:1.0:1787695917.942885:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:1.0:1787695917.942889:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695917.942890:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695917.942894:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695917.942897:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695917.942906:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695917.942914:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446544400. 00000100:00000001:1.0:1787695917.942922:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695917.942932:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446665f80 x1874531139544704/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695917.942943:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695917.942947:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695917.942952:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a0a2be0e] to pc [ptlrpcd_rcv+-1] req@ffff8b1446665f80 x1874531139544704/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695917.942972:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695917.942975:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695917.942979:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787695917.942980:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695917.942982:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695917.942985:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695917.942988:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3805 00000400:00000200:1.0:1787695917.942993:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695917.942997:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695917.942999:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787695917.943001:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1787695917.943003:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695917.943008:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787695917.943008:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1787695917.943013:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695917.943014:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787695917.943016:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695917.943020:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695917.943021:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:1.0:1787695917.943023:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695917.943031:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000100:00000040:2.0:1787695917.943039:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665f80 x1874531139544704/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787695917.943044:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:1.0:1787695917.943049:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:1.0:1787695917.943058:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695917.943059:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695917.943062:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787695917.943064:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695917.943066:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695917.943070:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:2.0:1787695917.943071:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:1.0:1787695917.943073:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 02000000:00000001:2.0:1787695917.943076:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1787695917.943077:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000001:2.0:1787695917.943084:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695917.943085:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695917.943091:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1787695917.943095:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139544704:192.168.201.140@tcp:250:kworker.0 02000000:00000001:1.0:1787695917.943097:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695917.943104:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.943105:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787695917.943107:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787695917.943111:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787695917.943114:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695917.943118:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695917.943119:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:1.0:1787695917.943122:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446667b80. 02000000:00000001:2.0:1787695917.943123:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1787695917.943127:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:2.0:1787695917.943129:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787695917.943133:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695917.943135:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695917.943137:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445662800. 00000100:00000001:1.0:1787695917.943138:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695917.943141:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695917.943143:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695917.943144:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695917.943152:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446547800. 00000400:00000200:2.0:1787695917.943153:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695917.943158:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1787695917.943160:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000100:00100000:1.0:1787695917.943166:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446667b80 x1874531139544832/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:2.0:1787695917.943170:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000001:1.0:1787695917.943179:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000200:2.0:1787695917.943181:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139544704, portal 25 00000100:00000001:1.0:1787695917.943182:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.943189:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:1.0:1787695917.943190:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006b2dcf49] to pc [ptlrpcd_rcv+-1] req@ffff8b1446667b80 x1874531139544832/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.943193:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929561984 : -128555779989632 : ffff8b1446665f80) 00000100:00000040:2.0:1787695917.943206:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665f80 x1874531139544704/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695942 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.943216:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695917.943219:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695917.943224:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695917.943231:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139544704, offset 0 00000100:00000001:1.0:1787695917.943233:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695917.943238:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1787695917.943243:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00080000:1.0:1787695917.943245:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787695917.943248:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695917.943250:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1787695917.943254:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695917.943255:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695917.943262:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3805 00000400:00000200:2.0:1787695917.943263:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695917.943271:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695917.943273:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695917.943276:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695917.943279:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695917.943281:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695917.943286:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17065:17065:256:4294967295] 192.168.201.140@tcp LPNI seq info [17064:17064:8:4294967295] 00000400:00000200:1.0:1787695917.943290:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000200:2.0:1787695917.943295:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:1.0:1787695917.943296:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695917.943302:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695917.943305:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695917.943310:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695917.943315:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:2.0:1787695917.943317:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000020:00000001:1.0:1787695917.943319:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695917.943322:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:1.0:1787695917.943326:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695917.943330:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000800:00000200:2.0:1787695917.943330:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000020:00000001:1.0:1787695917.943334:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:1.0:1787695917.943338:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000800:00000010:2.0:1787695917.943339:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000100:00000040:1.0:1787695917.943344:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695917.943349:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.0:1787695917.943354:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:1.0:1787695917.943355:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695917.943360:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695917.943405:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:1.0:1787695917.943410:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695917.943412:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695917.943421:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.0:1787695917.943423:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 02000000:00000001:1.0:1787695917.943425:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695917.943430:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695917.943434:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000200:2.1:1787695917.943435:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000008:00100000:1.0:1787695917.943437:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000800:00000200:2.1:1787695917.943442:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000008:00000001:1.0:1787695917.943443:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695917.943446:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695917.943453:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665500. 00000800:00000100:2.0:1787695917.943453:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000020:00000040:1.0:1787695917.943457:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:1.0:1787695917.943460:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695917.943462:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695917.943466:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000010:2.0:1787695917.943466:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 02000000:00000001:1.0:1787695917.943470:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695917.943474:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695917.943474:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000010:1.0:1787695917.943481:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446547c00. 00000400:00000200:2.0:1787695917.943483:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00000001:1.0:1787695917.943489:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695917.943490:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695917.943494:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695917.943498:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:1.0:1787695917.943499:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446665500 x1874531139544960/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1787695917.943504:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446665f80 x1874531139544704/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695942 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.943511:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695917.943518:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665f80 x1874531139544704/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695942 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695917.943527:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695917.943531:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.943537:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695917.943539:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004ae1679a] to pc [ptlrpcd_rcv+-1] req@ffff8b1446665500 x1874531139544960/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.943541:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695917.943549:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695917.943551:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787695917.943551:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695917.943557:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787695917.943559:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695917.943561:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1787695917.943562:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695917.943563:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695917.943568:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3805 00000100:00000001:2.0:1787695917.943569:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298489657 00000400:00000200:1.0:1787695917.943573:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695917.943577:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695917.943579:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787695917.943581:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298484537 00000100:00000040:1.0:1787695917.943589:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695917.943590:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298489657 00000100:00000001:1.0:1787695917.943592:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695917.943596:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695917.943597:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298484537 00000100:00000001:1.0:1787695917.943598:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695917.943601:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695917.943603:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:1.0:1787695917.943605:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787695917.943606:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1787695917.943607:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:1.0:1787695917.943612:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:2.0:1787695917.943614:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787695917.943616:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:2.0:1787695917.943620:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:1.0:1787695917.943622:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695917.943626:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1787695917.943630:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695917/real 1787695917] req@ffff8b1446665f80 x1874531139544704/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695917.943631:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000001:2.0:1787695917.943644:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695917.943646:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695917.943649:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695917.943652:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695917.943654:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695917.943656:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695917.943660:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665f80 x1874531139544704/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695917.943677:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665f80 x1874531139544704/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.943684:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695917.943686:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.943688:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695917.943690:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695917.943695:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446665f80 x1874531139544704/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.943702:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695917.943707:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665f80 x1874531139544704/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695917.943712:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:2.0:1787695917.943716:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000040:1.0:1787695917.943719:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00000001:2.0:1787695917.943720:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.943725:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695917.943726:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787695917.943729:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695917.943729:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695917.943732:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695917.943735:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695917.943736:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:1.0:1787695917.943741:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00080000:2.0:1787695917.943744:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695917.943746:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:1.0:1787695917.943748:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1787695917.943750:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665f80 x1874531139544704/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695917.943754:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695917.943760:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787695917.943765:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787695917.943765:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139544704:192.168.201.140@tcp:250:kworker.0 00000008:00000001:1.0:1787695917.943772:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.943774:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695917.943776:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.943777:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695917.943783:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665f80 x1874531139544704/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1787695917.943784:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446667480. 00000020:00000040:1.0:1787695917.943790:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:1.0:1787695917.943794:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695917.943796:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695917.943801:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695917.943804:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787695917.943807:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695917.943808:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695917.943810:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695917.943814:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445662800. 02000000:00000010:1.0:1787695917.943815:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446546000. 02000000:00000001:2.0:1787695917.943817:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695917.943819:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695917.943820:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:2.0:1787695917.943822:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695917.943824:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446544400. 00000100:00000001:1.0:1787695917.943829:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695917.943830:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695917.943834:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695917.943835:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665f80. 00000100:00100000:1.0:1787695917.943851:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446667480 x1874531139545088/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695917.943857:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695917.943859:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695917.943863:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695917.943863:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695917.943868:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.943872:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:1.0:1787695917.943874:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000390bc45a] to pc [ptlrpcd_rcv+-1] req@ffff8b1446667480 x1874531139545088/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.943875:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.943880:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695917.943882:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695917.943885:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695917.943891:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446667b80 x1874531139544832/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695917.943893:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695917.943899:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787695917.943904:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:1.0:1787695917.943905:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1787695917.943906:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695917.943909:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:2.0:1787695917.943910:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:1.0:1787695917.943915:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1787695917.943915:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695917.943920:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695917.943921:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:2.0:1787695917.943925:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446667b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139544832:192.168.201.140@tcp:38:kworker.0 00000100:00080000:1.0:1787695917.943929:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3805 00000100:00000001:2.0:1787695917.943930:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695917.943933:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695917.943936:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695917.943938:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:1.0:1787695917.943940:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000010:2.0:1787695917.943942:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445662800. 02000000:00000001:2.0:1787695917.943946:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695917.943947:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000200:2.0:1787695917.943952:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:1.0:1787695917.943953:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695917.943958:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000010:2.0:1787695917.943960:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000100:00000001:1.0:1787695917.943963:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:2.0:1787695917.943965:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000040:1.0:1787695917.943969:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000200:2.0:1787695917.943971:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139544832, portal 10 00000100:00000001:1.0:1787695917.943973:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:2.0:1787695917.943974:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695917.943976:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929569152 : -128555779982464 : ffff8b1446667b80) 00000020:00000001:1.0:1787695917.943980:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695917.943983:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000100:00000040:2.0:1787695917.943986:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446667b80 x1874531139544832/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695942 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787695917.943987:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695917.943991:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:1.0:1787695917.943997:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000001:2.0:1787695917.943997:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1787695917.944003:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:2.0:1787695917.944003:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695917.944006:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000040:1.0:1787695917.944011:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000200:2.0:1787695917.944011:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139544832, offset 0 00000100:00000001:1.0:1787695917.944018:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1787695917.944020:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695917.944023:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695917.944026:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695917.944029:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:1.0:1787695917.944035:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:2.0:1787695917.944035:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695917.944039:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000001:1.0:1787695917.944042:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:2.0:1787695917.944043:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695917.944049:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17066:17066:256:4294967295] 192.168.201.140@tcp LPNI seq info [17065:17065:8:4294967295] 00000020:00000040:1.0:1787695917.944050:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000400:00000200:2.0:1787695917.944056:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:1.0:1787695917.944059:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695917.944068:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695917.944068:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 02000000:00000001:1.0:1787695917.944073:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:2.0:1787695917.944077:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 02000000:00000001:1.0:1787695917.944079:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:2.0:1787695917.944080:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 02000000:00000001:1.0:1787695917.944086:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1787695917.944089:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:1.0:1787695917.944091:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000200:2.0:1787695917.944093:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000008:00100000:1.0:1787695917.944095:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000800:00000200:2.1:1787695917.944101:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000008:00000001:1.0:1787695917.944102:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695917.944106:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1787695917.944106:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1787695917.944113:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00000010:1.0:1787695917.944115:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446666d80. 00000800:00000010:2.0:1787695917.944116:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695917.944121:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:1.0:1787695917.944122:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 00000400:00000200:2.0:1787695917.944127:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 02000000:00000001:1.0:1787695917.944128:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1787695917.944130:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 02000000:00000001:1.0:1787695917.944132:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1787695917.944133:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:1.0:1787695917.944136:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695917.944137:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695917.944140:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.944142:0:2402:0:(events.c:34:request_out_callback()) Process entered 02000000:00000010:1.0:1787695917.944148:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446544c00. 00000100:00000200:2.0:1787695917.944148:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446667b80 x1874531139544832/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695942 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695917.944155:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.944159:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:1.0:1787695917.944165:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446666d80 x1874531139545216/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1787695917.944166:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446667b80 x1874531139544832/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695942 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.944194:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695917.944198:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695917.944198:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695917.944204:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.944204:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695917.944208:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000100:00000040:1.0:1787695917.944214:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa82a36] to pc [ptlrpcd_rcv+-1] req@ffff8b1446666d80 x1874531139545216/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787695917.944214:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695917.944219:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298479558 00000100:00000001:2.0:1787695917.944223:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298489657 00000100:00000001:2.0:1787695917.944227:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298479558 00000100:00000001:2.0:1787695917.944231:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298489657 00000100:00000001:2.0:1787695917.944236:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695917.944237:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.944239:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695917.944243:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787695917.944243:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695917.944247:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695917/real 1787695917] req@ffff8b1446667b80 x1874531139544832/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695917.944250:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787695917.944254:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000001:2.0:1787695917.944257:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695917.944258:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695917.944259:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695917.944260:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000100:00080000:1.0:1787695917.944264:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:2.0:1787695917.944264:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695917.944268:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787695917.944272:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:2.0:1787695917.944273:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446667b80 x1874531139544832/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695917.944280:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3805 00000100:00000200:2.0:1787695917.944287:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446667b80 x1874531139544832/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695917.944292:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695917.944301:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695917.944304:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695917.944308:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695917.944314:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695917.944319:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695917.944324:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:2.0:1787695917.944328:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:1.0:1787695917.944329:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695917.944333:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000100:00000001:2.0:1787695917.944333:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787695917.944338:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787695917.944339:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:1.0:1787695917.944343:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000100:00000001:2.0:1787695917.944343:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787695917.944350:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00100000:2.0:1787695917.944353:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446667b80 x1874531139544832/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787695917.944355:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000040:1.0:1787695917.944388:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695917.944389:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695917.944395:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695917.944400:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446667b80 x1874531139544832/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695917.944402:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695917.944407:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695917.944413:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00000001:2.0:1787695917.944417:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695917.944421:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695917.944422:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.944425:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695917.944428:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695917.944429:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.944431:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:1.0:1787695917.944433:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787695917.944436:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1787695917.944439:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787695917.944441:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 02000000:00000001:1.0:1787695917.944444:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695917.944449:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00080000:2.0:1787695917.944449:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000008:00100000:1.0:1787695917.944454:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787695917.944455:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00000001:1.0:1787695917.944461:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695917.944466:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695917.944466:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446667b80 x1874531139544832/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1787695917.944479:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665c00. 00000100:00100000:2.0:1787695917.944481:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446667b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139544832:192.168.201.140@tcp:38:kworker.0 00000020:00000040:1.0:1787695917.944485:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000100:00000001:2.0:1787695917.944487:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1787695917.944490:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695917.944494:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695917.944494:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665500 x1874531139544960/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695917.944498:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695917.944501:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695917.944505:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.944511:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000010:1.0:1787695917.944513:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545000. 00000100:00000001:2.0:1787695917.944515:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695917.944519:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695917.944526:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:1.0:1787695917.944527:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446665c00 x1874531139545344/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787695917.944530:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695917.944536:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695917.944545:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139544960:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695917.944547:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695917.944552:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.944553:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695917.944556:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695917.944559:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695917.944560:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d782cc4] to pc [ptlrpcd_rcv+-1] req@ffff8b1446665c00 x1874531139545344/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695917.944561:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695917.944565:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445661c00. 02000000:00000001:2.0:1787695917.944570:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695917.944579:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695917.944579:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:1.0:1787695917.944584:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:2.0:1787695917.944586:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000100:00080000:1.0:1787695917.944593:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000010:2.0:1787695917.944594:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000001:1.0:1787695917.944596:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695917.944599:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:2.0:1787695917.944602:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139544960, portal 10 00000100:00000001:1.0:1787695917.944604:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787695917.944607:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1787695917.944611:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3805 00000100:00000001:2.0:1787695917.944611:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929559296 : -128555779992320 : ffff8b1446665500) 00000400:00000200:1.0:1787695917.944620:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:2.0:1787695917.944625:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665500 x1874531139544960/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695942 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695917.944627:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695917.944633:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787695917.944636:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695917.944639:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695917.944641:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695917.944643:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139544960, offset 0 00000400:00000200:2.0:1787695917.944649:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:1.0:1787695917.944652:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695917.944656:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1787695917.944657:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:1.0:1787695917.944662:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:2.0:1787695917.944663:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:1.0:1787695917.944669:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:2.0:1787695917.944670:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:1.0:1787695917.944675:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:2.0:1787695917.944675:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695917.944678:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695917.944679:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695917.944682:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17067:17067:256:4294967295] 192.168.201.140@tcp LPNI seq info [17066:17066:8:4294967295] 00000020:00000040:1.0:1787695917.944686:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000400:00000200:2.0:1787695917.944687:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:1.0:1787695917.944692:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695917.944697:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000400:00000200:2.0:1787695917.944697:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000020:00000001:1.0:1787695917.944701:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000020:00000001:1.0:1787695917.944703:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:1.0:1787695917.944712:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000800:00000200:2.0:1787695917.944713:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787695917.944719:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:2.0:1787695917.944720:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000100:00000040:1.0:1787695917.944725:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000200:2.0:1787695917.944730:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:1.0:1787695917.944731:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000200:2.0:1787695917.944734:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000020:00000040:1.0:1787695917.944737:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000800:00000200:2.1:1787695917.944741:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695917.944745:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000100:00080000:1.0:1787695917.944746:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695917.944752:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1787695917.944752:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695917.944756:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 02000000:00000001:1.0:1787695917.944757:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1787695917.944760:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:1.0:1787695917.944763:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1787695917.944764:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 02000000:00000001:1.0:1787695917.944769:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695917.944771:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000100:00000001:1.0:1787695917.944774:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:2.0:1787695917.944774:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000008:00100000:1.0:1787695917.944779:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787695917.944779:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000008:00000001:1.0:1787695917.944785:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1787695917.944785:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446665500 x1874531139544960/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695942 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695917.944789:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695917.944797:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446666a00. 00000100:00000001:2.0:1787695917.944798:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000040:1.0:1787695917.944803:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000040:2.0:1787695917.944804:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665500 x1874531139544960/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695942 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695917.944810:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695917.944814:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.944815:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695917.944818:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787695917.944819:0:2402:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1787695917.944822:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787695917.944823:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695917.944826:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695917.944827:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 02000000:00000010:1.0:1787695917.944831:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545c00. 00000400:00000200:2.0:1787695917.944832:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695917.944835:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298478407 00000100:00000001:2.0:1787695917.944852:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298489657 00000100:00000001:1.0:1787695917.944853:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.944854:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298478407 00000100:00000001:2.0:1787695917.944859:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298489657 00000100:00000001:2.0:1787695917.944861:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695917.944863:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695917.944864:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446666a00 x1874531139545472/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695917.944865:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695917.944872:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695917/real 1787695917] req@ffff8b1446665500 x1874531139544960/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695917.944876:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695917.944881:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787695917.944887:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695917.944889:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695917.944892:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695917.944894:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000100:00000040:1.0:1787695917.944897:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000590093d2] to pc [ptlrpcd_rcv+-1] req@ffff8b1446666a00 x1874531139545472/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1787695917.944897:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695917.944902:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695917.944908:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665500 x1874531139544960/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695917.944917:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1787695917.944922:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665500 x1874531139544960/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695917.944923:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1787695917.944933:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:2.0:1787695917.944936:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:1.0:1787695917.944939:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787695917.944939:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.944943:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:1.0:1787695917.944947:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:2.0:1787695917.944947:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695917.944951:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446665500 x1874531139544960/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695917.944955:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3815) 00000100:00000001:2.0:1787695917.944957:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695917.944963:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665500 x1874531139544960/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.944968:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695917.944969:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.944970:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695917.944971:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.944973:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695917.944974:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695917.944976:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695917.944978:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695917.944980:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695917.944989:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665500 x1874531139544960/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695917.944994:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139544960:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695917.944998:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.944999:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695917.945002:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446667b80 x1874531139544832/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945006:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695917.945007:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695917.945009:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445662800. 02000000:00000001:2.0:1787695917.945011:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695917.945012:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695917.945013:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:2.0:1787695917.945014:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695917.945016:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446547800. 02000000:00000001:2.0:1787695917.945018:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695917.945019:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695917.945020:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446667b80. 00000100:00000001:2.0:1787695917.945023:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695917.945024:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.945025:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695917.945027:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665500 x1874531139544960/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945031:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695917.945031:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695917.945032:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445661c00. 02000000:00000001:2.0:1787695917.945034:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695917.945034:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695917.945035:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:2.0:1787695917.945036:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695917.945037:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446547c00. 02000000:00000001:2.0:1787695917.945038:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695917.945041:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695917.945042:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665500. 00000100:00000001:2.0:1787695917.945044:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695917.945045:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.945046:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.945049:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695917.945050:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.945051:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695917.945052:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695917.945053:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695917.945056:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446667480 x1874531139545088/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945060:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695917.945061:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695917.945063:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695917.945064:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695917.945066:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695917.945067:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446667480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139545088:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787695917.945069:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695917.945070:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695917.945071:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695917.945073:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695917.945074:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445661c00. 02000000:00000001:2.0:1787695917.945076:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695917.945078:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695917.945080:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695917.945082:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695917.945085:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139545088, portal 4 00000100:00000001:2.0:1787695917.945086:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695917.945087:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929567360 : -128555779984256 : ffff8b1446667480) 00000100:00000040:2.0:1787695917.945091:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446667480 x1874531139545088/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695942 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945095:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695917.945097:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695917.945101:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695917.945103:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139545088, offset 0 00000400:00000200:2.0:1787695917.945105:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695917.945108:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695917.945111:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695917.945113:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695917.945115:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695917.945117:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695917.945118:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695917.945120:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17068:17068:256:4294967295] 192.168.201.140@tcp LPNI seq info [17067:17067:8:4294967295] 00000400:00000200:2.0:1787695917.945123:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695917.945127:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695917.945130:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695917.945132:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695917.945135:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695917.945137:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1787695917.945140:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695917.945141:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1787695917.945144:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695917.945145:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695917.945147:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695917.945150:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695917.945152:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695917.945153:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695917.945154:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695917.945157:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446667480 x1874531139545088/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695942 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945162:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695917.945164:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446667480 x1874531139545088/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695942 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945168:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.945169:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695917.945175:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695917.945177:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:2.0:1787695917.945179:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695917.945181:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298478437 00000100:00000001:2.0:1787695917.945182:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298489657 00000100:00000001:2.0:1787695917.945184:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298478437 00000100:00000001:2.0:1787695917.945185:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298489657 00000100:00000001:2.0:1787695917.945187:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695917.945188:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.945189:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695917.945192:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695917/real 1787695917] req@ffff8b1446667480 x1874531139545088/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695917.945200:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695917.945202:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695917.945204:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695917.945206:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695917.945207:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695917.945210:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695917.945213:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446667480 x1874531139545088/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695917.945222:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446667480 x1874531139545088/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945229:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695917.945231:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.945233:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695917.945234:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695917.945238:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446667480 x1874531139545088/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945246:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695917.945249:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446667480 x1874531139545088/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945258:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695917.945260:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.945261:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695917.945262:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.945264:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695917.945265:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695917.945267:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695917.945269:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695917.945270:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695917.945273:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446667480 x1874531139545088/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695917.945278:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446667480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139545088:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787695917.945281:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695917.945283:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446666d80 x1874531139545216/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945287:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695917.945288:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695917.945289:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695917.945291:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695917.945293:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695917.945296:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446666d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139545216:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787695917.945299:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695917.945301:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695917.945303:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695917.945305:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695917.945308:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445662800. 02000000:00000001:2.0:1787695917.945310:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695917.945314:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695917.945321:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695917.945325:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695917.945328:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139545216, portal 4 00000100:00000001:2.0:1787695917.945331:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695917.945332:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929565568 : -128555779986048 : ffff8b1446666d80) 00000100:00000040:2.0:1787695917.945339:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446666d80 x1874531139545216/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695942 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945344:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695917.945345:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695917.945347:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695917.945349:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139545216, offset 0 00000400:00000200:2.0:1787695917.945352:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695917.945355:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695917.945357:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695917.945359:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695917.945361:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695917.945382:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695917.945384:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695917.945388:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17069:17069:256:4294967295] 192.168.201.140@tcp LPNI seq info [17068:17068:8:4294967295] 00000400:00000200:2.0:1787695917.945392:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695917.945399:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695917.945404:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695917.945408:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695917.945412:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695917.945413:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1787695917.945416:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695917.945418:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1787695917.945421:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695917.945422:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695917.945424:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695917.945427:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695917.945429:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695917.945431:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695917.945439:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695917.945444:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446666d80 x1874531139545216/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695942 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945452:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695917.945455:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446666d80 x1874531139545216/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695942 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945462:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.945463:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695917.945468:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695917.945470:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:2.0:1787695917.945474:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695917.945476:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298478110 00000100:00000001:2.0:1787695917.945479:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298489657 00000100:00000001:2.0:1787695917.945481:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298478110 00000100:00000001:2.0:1787695917.945484:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298489657 00000100:00000001:2.0:1787695917.945485:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695917.945487:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.945488:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695917.945492:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695917/real 1787695917] req@ffff8b1446666d80 x1874531139545216/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695917.945500:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695917.945502:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695917.945504:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695917.945506:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695917.945508:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695917.945522:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695917.945530:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446666d80 x1874531139545216/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695917.945538:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446666d80 x1874531139545216/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945553:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695917.945554:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.945555:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695917.945556:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695917.945559:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446666d80 x1874531139545216/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945564:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695917.945567:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446666d80 x1874531139545216/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945575:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695917.945577:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.945579:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695917.945580:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.945582:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695917.945584:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695917.945587:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695917.945590:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695917.945592:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695917.945597:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446666d80 x1874531139545216/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695917.945605:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446666d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139545216:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787695917.945610:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695917.945613:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665c00 x1874531139545344/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945621:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695917.945622:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695917.945628:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695917.945629:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695917.945631:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695917.945633:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139545344:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695917.945635:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695917.945637:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695917.945638:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695917.945639:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695917.945642:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445660800. 02000000:00000001:2.0:1787695917.945643:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695917.945646:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695917.945648:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695917.945650:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695917.945652:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139545344, portal 10 00000100:00000001:2.0:1787695917.945654:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695917.945655:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929561088 : -128555779990528 : ffff8b1446665c00) 00000100:00000040:2.0:1787695917.945659:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665c00 x1874531139545344/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695942 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945663:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695917.945665:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695917.945666:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695917.945668:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139545344, offset 0 00000400:00000200:2.0:1787695917.945670:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695917.945673:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695917.945675:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695917.945677:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695917.945679:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695917.945680:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695917.945682:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695917.945684:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17070:17070:256:4294967295] 192.168.201.140@tcp LPNI seq info [17069:17069:8:4294967295] 00000400:00000200:2.0:1787695917.945687:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695917.945691:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695917.945698:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695917.945700:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695917.945702:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695917.945704:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1787695917.945707:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695917.945712:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1787695917.945714:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695917.945716:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695917.945718:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695917.945720:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695917.945722:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695917.945723:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695917.945725:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695917.945728:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446665c00 x1874531139545344/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695942 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945733:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695917.945735:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665c00 x1874531139545344/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695942 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945739:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.945740:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695917.945742:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695917.945744:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:2.0:1787695917.945746:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695917.945747:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298478296 00000100:00000001:2.0:1787695917.945748:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298489657 00000100:00000001:2.0:1787695917.945750:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298478296 00000100:00000001:2.0:1787695917.945751:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298489657 00000100:00000001:2.0:1787695917.945753:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695917.945754:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.945755:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695917.945758:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695917/real 1787695917] req@ffff8b1446665c00 x1874531139545344/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695917.945767:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695917.945767:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695917.945769:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695917.945770:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695917.945771:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695917.945772:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695917.945774:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665c00 x1874531139545344/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695917.945780:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665c00 x1874531139545344/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945784:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695917.945785:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.945786:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695917.945786:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695917.945789:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446665c00 x1874531139545344/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945793:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695917.945795:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665c00 x1874531139545344/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945799:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695917.945801:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.945802:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695917.945803:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.945804:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695917.945805:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695917.945807:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695917.945808:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695917.945810:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695917.945817:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665c00 x1874531139545344/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695917.945822:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139545344:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695917.945825:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695917.945827:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446666a00 x1874531139545472/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945830:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695917.945831:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695917.945832:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695917.945834:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695917.945835:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695917.945849:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446666a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139545472:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695917.945852:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695917.945854:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695917.945855:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695917.945856:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695917.945858:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445662000. 02000000:00000001:2.0:1787695917.945860:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695917.945862:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695917.945864:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695917.945866:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695917.945868:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139545472, portal 10 00000100:00000001:2.0:1787695917.945870:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695917.945871:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929564672 : -128555779986944 : ffff8b1446666a00) 00000100:00000040:2.0:1787695917.945877:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446666a00 x1874531139545472/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695942 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945883:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695917.945886:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695917.945888:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695917.945891:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139545472, offset 0 00000400:00000200:2.0:1787695917.945901:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695917.945905:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695917.945908:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695917.945912:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695917.945914:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695917.945917:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695917.945919:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695917.945922:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17071:17071:256:4294967295] 192.168.201.140@tcp LPNI seq info [17070:17070:8:4294967295] 00000400:00000200:2.0:1787695917.945927:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695917.945933:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695917.945937:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695917.945941:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695917.945944:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695917.945947:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000800:00000200:2.1:1787695917.945952:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695917.945955:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000800:00000100:2.0:1787695917.945959:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695917.945961:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695917.945964:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695917.945967:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695917.945969:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695917.945970:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695917.945972:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695917.945975:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446666a00 x1874531139545472/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695942 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945980:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695917.945982:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446666a00 x1874531139545472/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695942 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.945987:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.945988:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695917.945990:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695917.945992:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:2.0:1787695917.945994:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695917.946004:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298478274 00000100:00000001:2.0:1787695917.946006:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298489657 00000100:00000001:2.0:1787695917.946008:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298478274 00000100:00000001:2.0:1787695917.946009:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298489657 00000100:00000001:2.0:1787695917.946010:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695917.946012:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.946013:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695917.946016:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695917/real 1787695917] req@ffff8b1446666a00 x1874531139545472/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695917.946032:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695917.946033:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695917.946035:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695917.946036:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695917.946038:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695917.946039:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695917.946041:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446666a00 x1874531139545472/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695917.946047:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446666a00 x1874531139545472/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.946051:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695917.946052:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.946053:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695917.946054:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695917.946056:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446666a00 x1874531139545472/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.946060:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695917.946063:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446666a00 x1874531139545472/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.946067:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695917.946072:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.946073:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695917.946073:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.946075:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695917.946076:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695917.946078:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695917.946080:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695917.946081:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695917.946083:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446666a00 x1874531139545472/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695917.946089:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446666a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139545472:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695917.946091:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.946092:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695917.946094:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446667480 x1874531139545088/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.946098:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695917.946099:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695917.946100:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445661c00. 02000000:00000001:2.0:1787695917.946102:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695917.946103:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695917.946104:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:2.0:1787695917.946106:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695917.946107:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446546000. 02000000:00000001:2.0:1787695917.946110:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695917.946111:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695917.946112:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446667480. 00000100:00000001:2.0:1787695917.946114:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695917.946115:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.946116:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695917.946118:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446666d80 x1874531139545216/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.946129:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695917.946130:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695917.946131:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445662800. 02000000:00000001:2.0:1787695917.946132:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695917.946133:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695917.946134:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:2.0:1787695917.946135:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695917.946136:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446544c00. 02000000:00000001:2.0:1787695917.946138:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695917.946138:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695917.946139:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446666d80. 00000100:00000001:2.0:1787695917.946140:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695917.946141:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.946142:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695917.946144:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665c00 x1874531139545344/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.946148:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695917.946149:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695917.946150:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445660800. 02000000:00000001:2.0:1787695917.946151:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695917.946151:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695917.946152:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:2.0:1787695917.946153:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695917.946154:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545000. 02000000:00000001:2.0:1787695917.946156:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695917.946156:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695917.946157:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665c00. 00000100:00000001:2.0:1787695917.946158:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695917.946159:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.946160:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695917.946162:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446666a00 x1874531139545472/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695942 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695917.946166:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695917.946166:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695917.946170:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445662000. 02000000:00000001:2.0:1787695917.946171:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695917.946172:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695917.946173:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:2.0:1787695917.946174:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695917.946175:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545c00. 02000000:00000001:2.0:1787695917.946177:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695917.946177:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695917.946178:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446666a00. 00000100:00000001:2.0:1787695917.946180:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695917.946182:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.946183:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695917.946187:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695917.946189:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695917.946191:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695917.946193:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695918.006415:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695918.006421:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695918.006423:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695918.006427:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695918.006430:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695918.006433:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695918.006435:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695918.006437:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695918.006439:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695918.646430:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695918.646435:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695918.646439:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695918.646440:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695918.646442:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695918.646443:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695918.646446:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695918.646447:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695918.646449:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695918.646450:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695918.646452:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695918.646453:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695918.646499:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695918.646502:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695918.646504:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695919.030469:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695919.030476:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695919.030478:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695919.030483:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695919.030486:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695919.030488:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695919.030490:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695919.030492:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695919.030494:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695919.670467:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695919.670472:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695919.670477:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695919.670478:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695919.670481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695919.670482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695919.670484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695919.670485:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695919.670487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695919.670488:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695919.670491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695919.670492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695919.670531:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695919.670535:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695919.670538:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695920.054460:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695920.054466:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695920.054468:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695920.054472:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695920.054475:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695920.054478:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695920.054479:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695920.054481:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695920.054483:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695920.694433:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695920.694454:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695920.694462:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695920.694463:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695920.694491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695920.694492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695920.694496:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695920.694498:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695920.694502:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695920.694503:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695920.694508:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695920.694509:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695920.694566:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695920.694570:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695920.694573:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695921.078439:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695921.078448:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695921.078449:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695921.078455:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695921.078457:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695921.078467:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695921.078469:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695921.078471:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695921.078473:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695921.718739:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695921.718745:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695921.718749:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695921.718750:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695921.718753:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695921.718754:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695921.718761:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695921.718762:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695921.718765:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695921.718767:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695921.718769:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695921.718789:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695921.718791:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695921.718792:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695921.718795:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695921.718797:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695921.718799:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695921.718800:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695921.718804:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695921.718805:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695921.718855:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695921.718859:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695921.718862:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695922.102443:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695922.102449:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695922.102450:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695922.102455:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695922.102457:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695922.102460:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695922.102462:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695922.102464:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695922.102466:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695922.742519:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695922.742528:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695922.742535:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695922.742536:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695922.742540:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695922.742542:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695922.742547:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695922.742571:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695922.742577:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695922.742578:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695922.742582:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695922.742584:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695922.742680:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695922.742684:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695922.742687:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787695923.062892:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695923.062908:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787695923.062911:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695923.062916:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695923.062922:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695923.062931:0:47074:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3810 00000400:00000200:2.0:1787695923.062943:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695923.062952:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695923.062956:0:47074:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695923.062959:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695923.062963:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695923.062965:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695923.062968:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695923.062973:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695923.062975:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:2.0:1787695923.062978:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695923.062982:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:2.0:1787695923.062985:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:2.0:1787695923.063007:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:2.0:1787695923.063012:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695923.063014:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695923.063016:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695923.063019:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695923.063023:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:2.0:1787695923.063026:0:47074:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695923.063029:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695923.063033:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695923.063038:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695923.063041:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695923.063044:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787695923.063047:0:47074:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695923.063069:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465af480. 00000020:00000040:2.0:1787695923.063074:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:2.0:1787695923.063078:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695923.063080:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695923.063083:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695923.063086:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695923.063091:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695923.063099:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445662000. 00000100:00000001:2.0:1787695923.063107:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695923.063117:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465af480 x1874531139545600/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695923.063179:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695923.063184:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695923.063189:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000071cfa937] to pc [ptlrpcd_rcv+-1] req@ffff8b14465af480 x1874531139545600/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695923.063210:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695923.063213:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695923.063219:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787695923.063228:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695923.063230:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695923.063235:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695923.063238:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3810 00000400:00000200:2.0:1787695923.063243:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695923.063246:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:2.0:1787695923.063248:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695923.063253:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695923.063254:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1787695923.063258:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695923.063260:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787695923.063265:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695923.063271:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695923.063273:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465af480 x1874531139545600/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695923.063275:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695923.063281:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695923.063284:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:2.0:1787695923.063288:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695923.063290:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:2.0:1787695923.063293:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:2.0:1787695923.063296:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:2.0:1787695923.063302:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695923.063305:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695923.063305:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695923.063309:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695923.063309:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695923.063315:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:1.0:1787695923.063317:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:2.0:1787695923.063318:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 02000000:00000001:1.0:1787695923.063320:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695923.063324:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695923.063326:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00100000:1.0:1787695923.063330:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139545600:192.168.201.140@tcp:250:kworker.0 00000100:00000001:2.0:1787695923.063332:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.063336:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695923.063338:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695923.063341:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695923.063345:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695923.063346:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695923.063348:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787695923.063353:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695923.063355:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546800. 00000100:00000001:2.0:1787695923.063357:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787695923.063358:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695923.063390:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695923.063394:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000008:00100000:2.0:1787695923.063397:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000010:1.0:1787695923.063403:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000008:00000001:2.0:1787695923.063403:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695923.063412:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139545600, portal 25 00000100:00000001:2.0:1787695923.063415:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.063416:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695923.063421:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928813696 : -128555780737920 : ffff8b14465af480) 00000100:00000010:2.0:1787695923.063423:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ac700. 00000100:00000040:1.0:1787695923.063428:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465af480 x1874531139545600/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1787695923.063430:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:2.0:1787695923.063438:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787695923.063441:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1787695923.063442:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695923.063447:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695923.063447:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695923.063450:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1787695923.063452:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 02000000:00000001:2.0:1787695923.063453:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695923.063458:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139545600, offset 0 02000000:00000010:2.0:1787695923.063461:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445660800. 00000100:00000001:2.0:1787695923.063465:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695923.063471:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ac700 x1874531139545728/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695923.063477:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695923.063483:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695923.063486:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695923.063489:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007c6208f8] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ac700 x1874531139545728/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695923.063490:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695923.063499:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:2.0:1787695923.063500:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695923.063504:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695923.063507:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:2.0:1787695923.063509:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787695923.063511:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695923.063513:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1787695923.063514:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:2.0:1787695923.063519:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1787695923.063522:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00080000:2.0:1787695923.063525:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3810 00000400:00000200:1.0:1787695923.063529:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695923.063532:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695923.063537:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17072:17072:256:4294967295] 192.168.201.140@tcp LPNI seq info [17071:17071:8:4294967295] 00000400:00000200:2.0:1787695923.063540:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695923.063544:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1787695923.063545:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:2.0:1787695923.063551:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695923.063553:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695923.063557:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695923.063562:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695923.063564:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000020:00000001:2.0:1787695923.063565:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000200:1.0:1787695923.063571:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695923.063577:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000020:00000040:2.0:1787695923.063578:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:2.0:1787695923.063581:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695923.063583:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000800:00000200:1.0:1787695923.063587:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000020:00000001:2.0:1787695923.063588:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000800:00000200:1.0:1787695923.063594:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000020:00000001:2.0:1787695923.063594:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:2.0:1787695923.063600:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000800:00000200:1.1:1787695923.063605:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:2.0:1787695923.063606:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695923.063612:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000200:1.1:1787695923.063612:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000100:00000001:2.0:1787695923.063617:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000100:1.0:1787695923.063622:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000020:00000040:2.0:1787695923.063622:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000800:00000010:1.0:1787695923.063628:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000100:00080000:2.0:1787695923.063631:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695923.063635:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695923.063638:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1787695923.063640:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:2.0:1787695923.063642:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695923.063647:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695923.063648:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00000001:2.0:1787695923.063654:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:1.0:1787695923.063655:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000008:00100000:2.0:1787695923.063658:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000010:1.0:1787695923.063659:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000008:00000001:2.0:1787695923.063665:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.063666:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787695923.063671:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695923.063672:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465af480 x1874531139545600/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1787695923.063678:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ae680. 00000100:00000001:1.0:1787695923.063683:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000040:2.0:1787695923.063684:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000100:00000040:1.0:1787695923.063688:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465af480 x1874531139545600/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695948 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695923.063689:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695923.063692:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695923.063696:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695923.063700:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695923.063703:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.063704:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.063707:0:2402:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000010:2.0:1787695923.063709:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445662800. 00000100:00000001:1.0:1787695923.063714:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695923.063714:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695923.063721:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00100000:2.0:1787695923.063724:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ae680 x1874531139545856/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695923.063726:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298494777 00000100:00000001:2.0:1787695923.063733:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695923.063736:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.063740:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298489657 00000100:00000040:2.0:1787695923.063763:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019831621] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ae680 x1874531139545856/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.063764:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298494777 00000100:00000001:1.0:1787695923.063771:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298489657 00000100:00000001:2.0:1787695923.063776:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.063777:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787695923.063781:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695923.063782:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.063787:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1787695923.063788:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787695923.063790:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695923.063793:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787695923.063794:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695923/real 1787695923] req@ffff8b14465af480 x1874531139545600/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695923.063796:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695923.063802:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3810 00000400:00000001:1.0:1787695923.063810:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:2.0:1787695923.063810:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000001:1.0:1787695923.063813:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1787695923.063814:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000010:1.0:1787695923.063817:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000100:00080000:2.0:1787695923.063818:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1787695923.063822:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000100:00000040:2.0:1787695923.063823:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000010:1.0:1787695923.063826:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:2.0:1787695923.063827:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695923.063832:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:2.0:1787695923.063832:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695923.063836:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000200:1.0:1787695923.063839:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465af480 x1874531139545600/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787695923.063839:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695923.063842:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:2.0:1787695923.063845:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695923.063848:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:2.0:1787695923.063853:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:2.0:1787695923.063855:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:2.0:1787695923.063857:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695923.063859:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695923.063860:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695923.063862:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000200:1.0:1787695923.063865:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465af480 x1874531139545600/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1787695923.063865:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:2.0:1787695923.063873:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695923.063878:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.063880:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695923.063882:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.063886:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1787695923.063887:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695923.063889:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787695923.063892:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1787695923.063895:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465af480 x1874531139545600/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695923.063896:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695923.063901:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695923.063905:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787695923.063907:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1787695923.063909:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695923.063912:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695923.063914:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465af480 x1874531139545600/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1787695923.063916:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ace00. 00000020:00000040:2.0:1787695923.063921:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000100:00000001:1.0:1787695923.063926:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1787695923.063927:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695923.063930:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.063931:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695923.063933:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695923.063934:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695923.063936:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695923.063939:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661c00. 00000100:00000001:1.0:1787695923.063940:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695923.063944:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695923.063946:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.063947:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695923.063951:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:2.0:1787695923.063955:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ace00 x1874531139545984/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787695923.063956:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695923.063965:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695923.063968:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695923.063969:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1787695923.063972:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695923.063976:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465af480 x1874531139545600/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695923.063980:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046eddc29] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ace00 x1874531139545984/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695923.063997:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139545600:192.168.201.140@tcp:250:kworker.0 00000100:00000001:2.0:1787695923.063998:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695923.064004:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695923.064006:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787695923.064007:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1787695923.064009:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787695923.064011:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695923.064013:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1787695923.064015:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465af480 x1874531139545600/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695923.064018:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695923.064024:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3810 00000100:00000001:1.0:1787695923.064026:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695923.064030:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:2.0:1787695923.064030:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000010:1.0:1787695923.064033:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546800. 00000400:00000200:2.0:1787695923.064035:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695923.064038:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1787695923.064039:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695923.064042:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695923.064043:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:1.0:1787695923.064049:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000040:2.0:1787695923.064051:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000010:1.0:1787695923.064052:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445662000. 00000100:00000001:2.0:1787695923.064054:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787695923.064058:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:2.0:1787695923.064058:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695923.064061:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787695923.064062:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000010:1.0:1787695923.064064:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465af480. 00000020:00000001:2.0:1787695923.064065:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695923.064068:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000020:00000040:2.0:1787695923.064068:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:2.0:1787695923.064069:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695923.064071:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1787695923.064073:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000100:00000001:1.0:1787695923.064075:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787695923.064076:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:2.0:1787695923.064080:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:2.0:1787695923.064083:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695923.064084:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695923.064085:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695923.064086:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695923.064087:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695923.064090:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1787695923.064091:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00000001:1.0:1787695923.064094:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1787695923.064097:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695923.064098:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695923.064101:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787695923.064102:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695923.064104:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695923.064106:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695923.064110:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695923.064111:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000040:1.0:1787695923.064113:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ac700 x1874531139545728/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787695923.064114:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787695923.064119:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.064126:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695923.064128:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695923.064131:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695923.064133:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695923.064133:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695923.064140:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695923.064143:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465af800. 00000100:00100000:1.0:1787695923.064145:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139545728:192.168.201.140@tcp:38:kworker.0 00000020:00000040:2.0:1787695923.064147:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:2.0:1787695923.064151:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787695923.064155:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695923.064155:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695923.064159:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695923.064161:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695923.064161:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695923.064165:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695923.064166:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695923.064169:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663800. 02000000:00000001:1.0:1787695923.064171:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695923.064175:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546800. 00000100:00000001:2.0:1787695923.064178:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695923.064180:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695923.064185:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465af800 x1874531139546112/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695923.064191:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695923.064196:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000100:00000001:2.0:1787695923.064201:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1787695923.064203:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000001:2.0:1787695923.064204:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695923.064209:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139545728, portal 10 00000100:00000040:2.0:1787695923.064211:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b410802f] to pc [ptlrpcd_rcv+-1] req@ffff8b14465af800 x1874531139546112/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.064213:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695923.064215:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928802048 : -128555780749568 : ffff8b14465ac700) 00000100:00000001:2.0:1787695923.064227:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695923.064228:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ac700 x1874531139545728/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695923.064234:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695923.064243:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1787695923.064245:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000040:1.0:1787695923.064251:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695923.064251:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695923.064255:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:1.0:1787695923.064257:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000001:2.0:1787695923.064264:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:1.0:1787695923.064265:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139545728, offset 0 00000100:00080000:2.0:1787695923.064271:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3810 00000400:00000200:1.0:1787695923.064277:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695923.064280:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695923.064286:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695923.064286:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695923.064291:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1787695923.064294:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000040:2.0:1787695923.064298:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695923.064301:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1787695923.064302:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695923.064307:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000040:2.0:1787695923.064308:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695923.064311:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695923.064313:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000020:00000001:2.0:1787695923.064315:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695923.064317:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000400:00000200:1.0:1787695923.064322:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000020:00000001:2.0:1787695923.064322:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695923.064327:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000400:00000200:1.0:1787695923.064328:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17073:17073:256:4294967295] 192.168.201.140@tcp LPNI seq info [17072:17072:8:4294967295] 00000020:00000001:2.0:1787695923.064335:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000200:1.0:1787695923.064336:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:2.0:1787695923.064342:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000200:1.0:1787695923.064345:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000040:2.0:1787695923.064351:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000800:00000200:1.0:1787695923.064353:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695923.064359:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000100:00000001:2.0:1787695923.064359:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.0:1787695923.064386:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695923.064390:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000100:00000040:2.0:1787695923.064391:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695923.064397:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000200:1.1:1787695923.064398:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695923.064404:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000020:00000040:2.0:1787695923.064405:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000800:00000100:1.0:1787695923.064410:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695923.064414:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000100:00080000:2.0:1787695923.064415:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:1.0:1787695923.064418:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1787695923.064422:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695923.064424:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 02000000:00000001:2.0:1787695923.064428:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1787695923.064429:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695923.064433:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 02000000:00000001:2.0:1787695923.064435:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695923.064436:0:2402:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1787695923.064440:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695923.064441:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ac700 x1874531139545728/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695923.064443:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695923.064447:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787695923.064453:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.064455:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787695923.064456:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695923.064461:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ac700 x1874531139545728/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695948 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1787695923.064461:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ad880. 00000020:00000040:2.0:1787695923.064467:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:2.0:1787695923.064472:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787695923.064473:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695923.064474:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.064477:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695923.064478:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695923.064481:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787695923.064482:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695923.064483:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695923.064487:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 02000000:00000010:2.0:1787695923.064488:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445662c00. 00000400:00000200:1.0:1787695923.064493:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695923.064495:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.064496:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298484679 00000100:00000001:1.0:1787695923.064501:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298494777 00000100:00100000:2.0:1787695923.064504:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ad880 x1874531139546240/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695923.064505:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298484679 00000100:00000001:1.0:1787695923.064509:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298494777 00000100:00000001:2.0:1787695923.064514:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695923.064517:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695923.064520:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.064521:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.064527:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:2.0:1787695923.064531:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000075b4d632] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ad880 x1874531139546240/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695923.064532:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695923/real 1787695923] req@ffff8b14465ac700 x1874531139545728/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695923.064552:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695923.064555:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695923.064556:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000100:00000001:2.0:1787695923.064559:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695923.064560:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695923.064562:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695923.064566:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:2.0:1787695923.064566:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1787695923.064571:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ac700 x1874531139545728/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695923.064573:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787695923.064576:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:1.0:1787695923.064581:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ac700 x1874531139545728/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695923.064585:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695923.064589:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695923.064592:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695923.064592:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695923.064596:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695923.064598:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787695923.064599:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3810 00000100:00100000:1.0:1787695923.064604:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ac700 x1874531139545728/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787695923.064608:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695923.064616:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695923.064617:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787695923.064620:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695923.064624:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ac700 x1874531139545728/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695923.064629:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695923.064631:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695923.064636:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695923.064636:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695923.064641:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695923.064643:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695923.064644:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695923.064645:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695923.064648:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695923.064650:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695923.064655:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000100:00080000:1.0:1787695923.064657:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000020:00000040:2.0:1787695923.064659:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000100:00080000:1.0:1787695923.064663:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000020:00000001:2.0:1787695923.064665:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695923.064666:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000040:2.0:1787695923.064670:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000100:00000040:1.0:1787695923.064671:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ac700 x1874531139545728/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787695923.064676:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00100000:1.0:1787695923.064682:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139545728:192.168.201.140@tcp:38:kworker.0 00000020:00000001:2.0:1787695923.064684:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:1.0:1787695923.064688:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695923.064695:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695923.064696:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ae680 x1874531139545856/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695923.064703:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695923.064706:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695923.064708:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695923.064709:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695923.064711:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787695923.064713:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:1.0:1787695923.064714:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1787695923.064717:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 02000000:00000001:1.0:1787695923.064718:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695923.064726:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139545856:192.168.201.140@tcp:38:kworker.0 00000100:00080000:2.0:1787695923.064726:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695923.064732:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.064733:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695923.064736:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695923.064738:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695923.064740:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695923.064741:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695923.064742:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787695923.064746:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695923.064747:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547400. 00000100:00000001:2.0:1787695923.064761:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787695923.064763:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787695923.064765:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1787695923.064770:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000008:00000001:2.0:1787695923.064771:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695923.064775:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000100:00000001:2.0:1787695923.064775:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695923.064780:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695923.064783:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139545856, portal 10 00000100:00000001:1.0:1787695923.064787:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695923.064788:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928810112 : -128555780741504 : ffff8b14465ae680) 00000100:00000010:2.0:1787695923.064789:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465aca80. 00000020:00000040:2.0:1787695923.064793:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000040:1.0:1787695923.064795:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ae680 x1874531139545856/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695923.064797:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695923.064800:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695923.064804:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695923.064807:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695923.064809:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695923.064813:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663c00. 00000100:00000001:2.0:1787695923.064819:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.064825:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1787695923.064829:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465aca80 x1874531139546368/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1787695923.064832:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695923.064837:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695923.064842:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139545856, offset 0 00000100:00000001:2.0:1787695923.064843:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695923.064847:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695923.064848:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:2.0:1787695923.064854:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004fad31e4] to pc [ptlrpcd_rcv+-1] req@ffff8b14465aca80 x1874531139546368/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695923.064855:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695923.064863:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695923.064870:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1787695923.064870:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695923.064874:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000040:2.0:1787695923.064875:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695923.064879:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000040:2.0:1787695923.064881:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1787695923.064884:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000040:2.0:1787695923.064886:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695923.064890:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17074:17074:256:4294967295] 192.168.201.140@tcp LPNI seq info [17073:17073:8:4294967295] 00000100:00000040:2.0:1787695923.064891:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1787695923.064898:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:2.0:1787695923.064899:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3820) 00000400:00000200:1.0:1787695923.064907:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695923.064910:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695923.064913:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695923.064916:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695923.064918:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1787695923.064923:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695923.064925:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1787695923.064928:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695923.064929:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695923.064932:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695923.064934:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695923.064936:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695923.064937:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695923.064939:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695923.064942:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ae680 x1874531139545856/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.064948:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695923.064950:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ae680 x1874531139545856/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695948 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.064963:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.064964:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695923.064967:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695923.064969:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787695923.064972:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695923.064973:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298483527 00000100:00000001:1.0:1787695923.064975:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298494777 00000100:00000001:1.0:1787695923.064977:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298483527 00000100:00000001:1.0:1787695923.064978:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298494777 00000100:00000001:1.0:1787695923.064979:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695923.064981:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.064982:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695923.064984:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695923/real 1787695923] req@ffff8b14465ae680 x1874531139545856/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695923.064990:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695923.064991:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695923.064992:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695923.064994:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695923.064995:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695923.064997:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695923.064999:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ae680 x1874531139545856/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695923.065004:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ae680 x1874531139545856/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065008:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695923.065009:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.065010:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695923.065011:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695923.065013:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ae680 x1874531139545856/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065021:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695923.065024:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ae680 x1874531139545856/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065028:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695923.065030:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.065031:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695923.065031:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.065033:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695923.065034:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695923.065035:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695923.065037:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695923.065038:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695923.065041:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ae680 x1874531139545856/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695923.065046:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139545856:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695923.065049:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695923.065051:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ace00 x1874531139545984/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065056:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695923.065056:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695923.065058:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695923.065059:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695923.065060:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695923.065062:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139545984:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695923.065064:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695923.065065:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695923.065069:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695923.065071:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695923.065073:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787695923.065075:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695923.065077:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695923.065079:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695923.065082:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695923.065084:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139545984, portal 4 00000100:00000001:1.0:1787695923.065085:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695923.065086:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928803840 : -128555780747776 : ffff8b14465ace00) 00000100:00000040:1.0:1787695923.065091:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ace00 x1874531139545984/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065096:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695923.065097:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695923.065099:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695923.065100:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139545984, offset 0 00000400:00000200:1.0:1787695923.065103:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695923.065105:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695923.065107:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695923.065109:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695923.065111:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695923.065113:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695923.065114:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695923.065117:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17075:17075:256:4294967295] 192.168.201.140@tcp LPNI seq info [17074:17074:8:4294967295] 00000400:00000200:1.0:1787695923.065119:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695923.065123:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695923.065127:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695923.065130:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695923.065135:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695923.065138:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1787695923.065142:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695923.065144:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1787695923.065149:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695923.065156:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695923.065160:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695923.065164:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695923.065167:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695923.065169:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695923.065172:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695923.065176:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ace00 x1874531139545984/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065185:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695923.065188:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ace00 x1874531139545984/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695948 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065193:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.065194:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695923.065197:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695923.065200:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695923.065202:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695923.065204:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298483557 00000100:00000001:1.0:1787695923.065206:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298494777 00000100:00000001:1.0:1787695923.065207:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298483557 00000100:00000001:1.0:1787695923.065209:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298494777 00000100:00000001:1.0:1787695923.065210:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695923.065212:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.065213:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695923.065215:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695923/real 1787695923] req@ffff8b14465ace00 x1874531139545984/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695923.065220:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695923.065221:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695923.065222:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695923.065223:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695923.065224:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695923.065232:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695923.065236:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ace00 x1874531139545984/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695923.065245:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ace00 x1874531139545984/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065253:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695923.065254:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.065256:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695923.065258:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695923.065262:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ace00 x1874531139545984/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065270:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695923.065274:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ace00 x1874531139545984/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065280:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695923.065282:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.065284:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695923.065285:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.065286:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695923.065287:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695923.065289:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695923.065291:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695923.065292:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695923.065295:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ace00 x1874531139545984/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695923.065301:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139545984:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695923.065303:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.065305:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695923.065311:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ac700 x1874531139545728/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065316:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695923.065317:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695923.065319:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546800. 02000000:00000001:1.0:1787695923.065321:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695923.065322:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695923.065323:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695923.065324:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695923.065326:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445660800. 02000000:00000001:1.0:1787695923.065328:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695923.065329:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695923.065330:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ac700. 00000100:00000001:1.0:1787695923.065333:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695923.065334:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.065336:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695923.065338:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ae680 x1874531139545856/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065342:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695923.065343:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695923.065344:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547400. 02000000:00000001:1.0:1787695923.065347:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695923.065347:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695923.065348:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695923.065349:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695923.065350:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445662800. 02000000:00000001:1.0:1787695923.065351:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695923.065352:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695923.065353:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ae680. 00000100:00000001:1.0:1787695923.065354:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695923.065355:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.065356:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695923.065358:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ace00 x1874531139545984/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065361:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695923.065384:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695923.065387:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787695923.065391:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695923.065393:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695923.065396:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695923.065398:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695923.065400:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661c00. 02000000:00000001:1.0:1787695923.065403:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695923.065405:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695923.065406:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ace00. 00000100:00000001:1.0:1787695923.065410:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695923.065412:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.065414:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.065417:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695923.065419:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.065421:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695923.065425:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695923.065427:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695923.065433:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465af800 x1874531139546112/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065438:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695923.065440:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695923.065442:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695923.065444:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695923.065446:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695923.065448:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139546112:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695923.065450:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695923.065452:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695923.065453:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695923.065454:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695923.065456:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787695923.065457:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695923.065460:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695923.065462:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695923.065464:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695923.065467:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139546112, portal 4 00000100:00000001:1.0:1787695923.065472:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695923.065473:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928814592 : -128555780737024 : ffff8b14465af800) 00000100:00000040:1.0:1787695923.065478:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465af800 x1874531139546112/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065483:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695923.065484:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695923.065486:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695923.065488:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139546112, offset 0 00000400:00000200:1.0:1787695923.065491:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695923.065494:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695923.065496:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695923.065498:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695923.065500:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695923.065502:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695923.065504:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695923.065506:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17076:17076:256:4294967295] 192.168.201.140@tcp LPNI seq info [17075:17075:8:4294967295] 00000400:00000200:1.0:1787695923.065509:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695923.065513:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695923.065516:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695923.065518:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695923.065521:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695923.065523:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1787695923.065526:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695923.065528:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1787695923.065531:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695923.065532:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695923.065535:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695923.065537:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695923.065539:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695923.065540:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695923.065542:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695923.065544:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465af800 x1874531139546112/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065558:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695923.065560:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465af800 x1874531139546112/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695948 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065565:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.065566:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695923.065568:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695923.065570:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695923.065573:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695923.065575:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298483230 00000100:00000001:1.0:1787695923.065577:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298494777 00000100:00000001:1.0:1787695923.065578:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298483230 00000100:00000001:1.0:1787695923.065580:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298494777 00000100:00000001:1.0:1787695923.065581:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695923.065583:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.065584:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695923.065586:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695923/real 1787695923] req@ffff8b14465af800 x1874531139546112/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695923.065591:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695923.065591:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695923.065592:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695923.065594:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695923.065595:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695923.065597:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695923.065599:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465af800 x1874531139546112/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695923.065604:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465af800 x1874531139546112/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065608:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695923.065612:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.065613:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695923.065614:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695923.065617:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465af800 x1874531139546112/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065622:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695923.065624:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465af800 x1874531139546112/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065628:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695923.065629:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.065631:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695923.065632:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.065633:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695923.065634:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695923.065636:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695923.065638:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695923.065639:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695923.065642:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465af800 x1874531139546112/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695923.065646:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139546112:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695923.065648:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695923.065650:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ad880 x1874531139546240/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065654:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695923.065655:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695923.065656:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695923.065657:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695923.065659:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695923.065663:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139546240:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695923.065666:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695923.065667:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695923.065668:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695923.065670:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695923.065672:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547400. 02000000:00000001:1.0:1787695923.065673:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695923.065675:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695923.065677:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695923.065679:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695923.065681:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139546240, portal 10 00000100:00000001:1.0:1787695923.065682:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695923.065683:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928806528 : -128555780745088 : ffff8b14465ad880) 00000100:00000040:1.0:1787695923.065688:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ad880 x1874531139546240/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065692:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695923.065694:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695923.065695:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695923.065697:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139546240, offset 0 00000400:00000200:1.0:1787695923.065699:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695923.065701:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695923.065703:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695923.065705:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695923.065707:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695923.065708:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695923.065710:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695923.065712:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17077:17077:256:4294967295] 192.168.201.140@tcp LPNI seq info [17076:17076:8:4294967295] 00000400:00000200:1.0:1787695923.065715:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695923.065719:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695923.065722:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695923.065724:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695923.065730:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695923.065732:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1787695923.065735:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695923.065736:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1787695923.065739:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695923.065741:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695923.065743:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695923.065745:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695923.065747:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695923.065763:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695923.065765:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695923.065768:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ad880 x1874531139546240/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065774:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695923.065776:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ad880 x1874531139546240/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695948 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065781:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.065782:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695923.065784:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695923.065787:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695923.065789:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695923.065790:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298483416 00000100:00000001:1.0:1787695923.065792:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298494777 00000100:00000001:1.0:1787695923.065794:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298483416 00000100:00000001:1.0:1787695923.065796:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298494777 00000100:00000001:1.0:1787695923.065797:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695923.065799:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.065800:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695923.065802:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695923/real 1787695923] req@ffff8b14465ad880 x1874531139546240/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695923.065807:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695923.065812:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695923.065814:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695923.065815:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695923.065816:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695923.065818:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695923.065821:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ad880 x1874531139546240/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695923.065826:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ad880 x1874531139546240/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065830:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695923.065831:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.065832:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695923.065833:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695923.065835:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ad880 x1874531139546240/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065839:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695923.065841:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ad880 x1874531139546240/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065845:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695923.065847:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.065848:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695923.065849:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.065850:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695923.065852:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695923.065853:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695923.065855:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695923.065856:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695923.065859:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ad880 x1874531139546240/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695923.065871:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139546240:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695923.065874:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695923.065876:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465aca80 x1874531139546368/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065881:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695923.065882:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695923.065884:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695923.065886:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695923.065888:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695923.065891:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139546368:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695923.065894:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695923.065896:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695923.065898:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695923.065900:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695923.065903:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546800. 02000000:00000001:1.0:1787695923.065905:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695923.065909:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695923.065911:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695923.065915:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695923.065919:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139546368, portal 10 00000100:00000001:1.0:1787695923.065921:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695923.065923:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928802944 : -128555780748672 : ffff8b14465aca80) 00000100:00000040:1.0:1787695923.065929:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465aca80 x1874531139546368/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.065937:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695923.065939:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695923.065941:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695923.065943:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139546368, offset 0 00000400:00000200:1.0:1787695923.065946:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695923.065953:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695923.065955:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695923.065958:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695923.065960:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695923.065961:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695923.065963:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695923.065965:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17078:17078:256:4294967295] 192.168.201.140@tcp LPNI seq info [17077:17077:8:4294967295] 00000400:00000200:1.0:1787695923.065968:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695923.065973:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695923.065976:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695923.065978:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695923.065981:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695923.065983:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1787695923.065986:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695923.065988:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1787695923.065991:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695923.065993:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695923.065995:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695923.065997:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695923.065999:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695923.066000:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695923.066002:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695923.066004:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465aca80 x1874531139546368/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695948 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.066010:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695923.066012:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465aca80 x1874531139546368/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695948 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.066017:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.066018:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695923.066020:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695923.066022:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695923.066024:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695923.066029:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298483394 00000100:00000001:1.0:1787695923.066031:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298494777 00000100:00000001:1.0:1787695923.066033:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298483394 00000100:00000001:1.0:1787695923.066035:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298494777 00000100:00000001:1.0:1787695923.066036:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695923.066038:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.066039:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695923.066042:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695923/real 1787695923] req@ffff8b14465aca80 x1874531139546368/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695923.066048:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695923.066049:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695923.066050:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695923.066052:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695923.066053:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695923.066054:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695923.066056:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465aca80 x1874531139546368/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695923.066062:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465aca80 x1874531139546368/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.066066:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695923.066067:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.066068:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695923.066069:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695923.066071:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465aca80 x1874531139546368/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.066075:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695923.066077:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465aca80 x1874531139546368/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.066081:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695923.066086:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.066087:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695923.066088:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.066089:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695923.066090:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695923.066092:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695923.066094:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695923.066095:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695923.066098:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465aca80 x1874531139546368/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695923.066103:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139546368:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695923.066106:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.066116:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695923.066118:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465af800 x1874531139546112/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.066127:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695923.066128:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695923.066130:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787695923.066133:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695923.066135:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695923.066136:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695923.066139:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695923.066140:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663800. 02000000:00000001:1.0:1787695923.066143:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695923.066145:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695923.066146:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465af800. 00000100:00000001:1.0:1787695923.066149:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695923.066151:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.066152:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695923.066156:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ad880 x1874531139546240/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.066176:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695923.066177:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695923.066179:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547400. 02000000:00000001:1.0:1787695923.066182:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695923.066183:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695923.066184:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695923.066185:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695923.066186:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445662c00. 02000000:00000001:1.0:1787695923.066188:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695923.066189:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695923.066190:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ad880. 00000100:00000001:1.0:1787695923.066191:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695923.066192:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.066193:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695923.066196:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465aca80 x1874531139546368/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695948 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695923.066201:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695923.066201:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695923.066202:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546800. 02000000:00000001:1.0:1787695923.066204:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695923.066205:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695923.066206:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695923.066207:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695923.066208:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663c00. 02000000:00000001:1.0:1787695923.066209:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695923.066210:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695923.066211:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465aca80. 00000100:00000001:1.0:1787695923.066212:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695923.066213:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.066214:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695923.066217:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695923.066218:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695923.066219:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695923.066220:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695923.126472:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695923.126478:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695923.126480:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695923.126484:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695923.126487:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695923.126490:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695923.126492:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695923.126494:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695923.126495:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695923.766484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695923.766518:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695923.766521:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695923.766524:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695923.766529:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695923.766532:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695923.766534:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695923.766536:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695923.766542:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695923.766544:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695923.766545:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695923.766547:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695923.766548:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695923.766550:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695923.766552:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695923.766554:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695923.766560:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695923.766562:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695923.766567:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695923.766568:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695923.766573:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695923.766574:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695923.766578:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695923.766580:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695923.766657:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695923.766663:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695923.766667:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695924.150448:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695924.150455:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695924.150458:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695924.150463:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695924.150467:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695924.150471:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695924.150474:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695924.150476:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695924.150479:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695924.790513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695924.790520:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695924.790525:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695924.790526:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695924.790529:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695924.790530:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695924.790534:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695924.790535:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695924.790537:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695924.790538:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695924.790541:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695924.790542:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695924.790631:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695924.790637:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695924.790640:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695925.174517:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695925.174550:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695925.174552:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695925.174558:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695925.174561:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695925.174565:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695925.174567:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695925.174570:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695925.174572:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695925.814530:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695925.814538:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695925.814543:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695925.814568:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695925.814575:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695925.814577:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695925.814581:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695925.814583:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695925.814587:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695925.814589:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695925.814593:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695925.814594:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695925.814636:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695925.814640:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695925.814643:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695926.198440:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695926.198445:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695926.198447:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695926.198451:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695926.198454:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695926.198457:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695926.198460:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695926.198462:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695926.198465:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695926.838440:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695926.838448:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695926.838453:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695926.838455:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695926.838458:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695926.838461:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695926.838464:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695926.838471:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695926.838473:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695926.838474:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695926.838478:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695926.838480:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695926.838482:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695926.838483:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695926.838487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695926.838488:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695926.838489:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695926.838490:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695926.838491:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695926.838492:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695926.838493:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695926.838493:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695926.838494:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695926.838496:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695926.838498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695926.838499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695926.838502:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695926.838504:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695926.838505:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695926.838506:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695926.838529:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695926.838530:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695926.838532:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695926.838533:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695926.838534:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695926.838536:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695926.838650:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695926.838656:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695926.838660:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695927.222445:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695927.222453:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695927.222456:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695927.222462:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695927.222465:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695927.222469:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695927.222472:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695927.222475:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695927.222477:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695927.862444:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695927.862452:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695927.862457:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695927.862458:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695927.862462:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695927.862463:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695927.862466:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695927.862467:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695927.862470:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695927.862471:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695927.862474:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695927.862475:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695927.862548:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695927.862555:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695927.862559:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787695928.182505:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695928.182516:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787695928.182518:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695928.182522:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695928.182526:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695928.182545:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3815 00000400:00000200:3.0:1787695928.182554:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695928.182574:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695928.182578:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695928.182582:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695928.182585:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695928.182588:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695928.182591:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695928.182595:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695928.182597:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:3.0:1787695928.182600:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695928.182604:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787695928.182606:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787695928.182609:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787695928.182612:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695928.182614:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695928.182616:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695928.182618:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695928.182621:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787695928.182624:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695928.182627:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695928.182630:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695928.182633:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695928.182636:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695928.182638:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787695928.182641:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695928.182647:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6680. 00000020:00000040:3.0:1787695928.182651:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787695928.182655:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695928.182657:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695928.182659:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695928.182675:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695928.182679:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695928.182688:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675800. 00000100:00000001:3.0:1787695928.182701:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695928.182712:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6680 x1874531139546496/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695928.182723:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695928.182727:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695928.182732:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f6d2abf9] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6680 x1874531139546496/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695928.182779:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695928.182783:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695928.182787:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695928.182788:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695928.182790:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695928.182793:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695928.182797:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3815 00000400:00000200:3.0:1787695928.182802:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695928.182805:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695928.182808:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695928.182812:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695928.182815:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695928.182818:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695928.182821:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695928.182826:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695928.182827:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:3.0:1787695928.182829:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695928.182830:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:3.0:1787695928.182832:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000100:00000001:1.0:1787695928.182836:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1787695928.182837:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000001:1.0:1787695928.182841:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1787695928.182842:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:3.0:1787695928.182852:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695928.182853:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6680 x1874531139546496/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695928.182855:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695928.182868:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695928.182869:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695928.182872:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695928.182873:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695928.182878:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 02000000:00000001:1.0:1787695928.182880:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1787695928.182885:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000001:1.0:1787695928.182886:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695928.182893:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695928.182894:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695928.182898:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1787695928.182903:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139546496:192.168.201.140@tcp:250:kworker.0 02000000:00000001:3.0:1787695928.182906:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695928.182924:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787695928.182926:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695928.182930:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787695928.182931:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:3.0:1787695928.182935:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787695928.182936:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695928.182941:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:3.0:1787695928.182942:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695928.182946:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695928.182949:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547400. 00000100:00000010:3.0:1787695928.182952:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5880. 02000000:00000001:1.0:1787695928.182953:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695928.182957:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695928.182961:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695928.182963:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695928.182965:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695928.182967:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695928.182969:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695928.182976:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675400. 00000400:00000200:1.0:1787695928.182977:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695928.182981:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695928.182988:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5880 x1874531139546624/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1787695928.182992:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000100:00000001:3.0:1787695928.182998:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695928.183001:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695928.183002:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000040:3.0:1787695928.183004:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000827e7dbe] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5880 x1874531139546624/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695928.183027:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139546496, portal 25 00000100:00000001:3.0:1787695928.183030:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.183033:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787695928.183034:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695928.183038:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930153600 : -128555779398016 : ffff8b14466f6680) 00000100:00080000:3.0:1787695928.183040:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695928.183045:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:1.0:1787695928.183051:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6680 x1874531139546496/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695928.183057:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695928.183065:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695928.183071:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787695928.183072:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3815 00000100:00000040:1.0:1787695928.183079:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000200:3.0:1787695928.183083:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695928.183101:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000400:00000200:3.0:1787695928.183106:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000200:1.0:1787695928.183109:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139546496, offset 0 00000100:00080000:3.0:1787695928.183110:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695928.183116:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695928.183121:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695928.183121:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695928.183126:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695928.183130:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695928.183135:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:3.0:1787695928.183136:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695928.183140:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:3.0:1787695928.183142:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695928.183144:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000400:00000200:1.0:1787695928.183147:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000020:00000001:3.0:1787695928.183148:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:3.0:1787695928.183154:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000400:00000200:1.0:1787695928.183156:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1787695928.183161:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695928.183166:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:3.0:1787695928.183167:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695928.183173:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695928.183175:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000001:3.0:1787695928.183179:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695928.183181:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000020:00000040:3.0:1787695928.183184:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000400:00000200:1.0:1787695928.183190:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17079:17079:256:4294967295] 192.168.201.140@tcp LPNI seq info [17078:17078:8:4294967295] 00000100:00080000:3.0:1787695928.183193:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695928.183199:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695928.183201:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787695928.183203:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695928.183209:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695928.183214:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695928.183218:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:3.0:1787695928.183218:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695928.183222:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695928.183226:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695928.183230:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695928.183230:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1787695928.183236:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000100:00000010:3.0:1787695928.183239:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4a80. 00000020:00000040:3.0:1787695928.183243:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000800:00000200:1.0:1787695928.183245:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:3.0:1787695928.183246:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695928.183251:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695928.183252:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 6 secs later) 00000100:00000001:3.0:1787695928.183253:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695928.183256:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695928.183260:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1787695928.183260:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695928.183266:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 6 secs later) 02000000:00000010:3.0:1787695928.183267:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 00000800:00000100:1.0:1787695928.183272:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695928.183272:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1787695928.183275:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695928.183280:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00100000:3.0:1787695928.183280:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4a80 x1874531139546752/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695928.183307:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1787695928.183313:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695928.183315:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695928.183317:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000100:00000040:3.0:1787695928.183319:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ad0d67] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4a80 x1874531139546752/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1787695928.183321:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:3.0:1787695928.183326:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695928.183329:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695928.183334:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787695928.183335:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787695928.183337:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1787695928.183337:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695928.183343:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695928.183347:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3815 00000100:00000200:1.0:1787695928.183348:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6680 x1874531139546496/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695928.183352:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695928.183358:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695928.183362:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695928.183366:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695928.183369:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695928.183372:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695928.183374:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787695928.183375:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695928.183378:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000040:1.0:1787695928.183382:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6680 x1874531139546496/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695953 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1787695928.183387:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:3.0:1787695928.183391:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695928.183392:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695928.183393:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000100:00000001:1.0:1787695928.183395:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1787695928.183397:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:3.0:1787695928.183400:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:1.0:1787695928.183403:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695928.183404:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695928.183408:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695928.183410:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695928.183411:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695928.183413:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:3.0:1787695928.183415:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695928.183417:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695928.183418:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695928.183421:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695928.183422:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695928.183434:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695928.183437:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298499897 00000100:00000001:1.0:1787695928.183444:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298494777 00000100:00000001:3.0:1787695928.183450:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695928.183451:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298499897 00000008:00100000:3.0:1787695928.183452:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787695928.183457:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.183459:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298494777 00000100:00000001:3.0:1787695928.183459:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695928.183463:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4000. 00000100:00000001:1.0:1787695928.183466:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1787695928.183466:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:3.0:1787695928.183472:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787695928.183473:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695928.183474:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.183478:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787695928.183478:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695928.183480:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695928.183483:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695928.183486:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676000. 00000100:00100000:1.0:1787695928.183489:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695928/real 1787695928] req@ffff8b14466f6680 x1874531139546496/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695928.183490:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695928.183498:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4000 x1874531139546880/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695928.183508:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695928.183509:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787695928.183516:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:3.0:1787695928.183518:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5a8f356] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4000 x1874531139546880/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695928.183519:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695928.183523:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695928.183528:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695928.183531:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:3.0:1787695928.183532:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.183537:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:3.0:1787695928.183537:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695928.183541:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000200:1.0:1787695928.183544:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6680 x1874531139546496/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695928.183544:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695928.183545:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695928.183549:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695928.183553:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3815 00000400:00000200:3.0:1787695928.183560:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000200:1.0:1787695928.183563:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6680 x1874531139546496/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695928.183566:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695928.183570:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695928.183575:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695928.183577:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695928.183578:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695928.183580:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695928.183584:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695928.183585:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000100:00000001:1.0:1787695928.183588:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:3.0:1787695928.183588:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695928.183590:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:3.0:1787695928.183592:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:3.0:1787695928.183593:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:3.0:1787695928.183597:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695928.183598:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695928.183600:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695928.183602:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:3.0:1787695928.183604:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695928.183605:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695928.183608:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00100000:1.0:1787695928.183612:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6680 x1874531139546496/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1787695928.183613:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:3.0:1787695928.183619:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695928.183623:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.183626:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787695928.183631:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1787695928.183632:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6680 x1874531139546496/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695928.183636:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695928.183640:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.183645:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787695928.183645:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695928.183648:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787695928.183650:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1787695928.183653:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.183654:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695928.183656:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695928.183657:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.183659:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695928.183676:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695928.183679:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000010:3.0:1787695928.183679:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4700. 00000020:00000040:3.0:1787695928.183684:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 00000100:00080000:1.0:1787695928.183686:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 02000000:00000001:3.0:1787695928.183688:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787695928.183689:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1787695928.183691:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695928.183694:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695928.183696:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1787695928.183697:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6680 x1874531139546496/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695928.183698:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695928.183704:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 00000100:00100000:1.0:1787695928.183709:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139546496:192.168.201.140@tcp:250:kworker.0 00000100:00000001:3.0:1787695928.183709:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.183716:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787695928.183717:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4700 x1874531139547008/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695928.183719:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695928.183725:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6680 x1874531139546496/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695928.183733:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695928.183736:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.183738:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695928.183741:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:3.0:1787695928.183742:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008818c2cc] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4700 x1874531139547008/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1787695928.183744:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547400. 02000000:00000001:1.0:1787695928.183750:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695928.183752:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:3.0:1787695928.183752:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695928.183755:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695928.183758:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000020:00000040:1.0:1787695928.183762:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000100:00000001:3.0:1787695928.183762:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695928.183763:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:1.0:1787695928.183766:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:3.0:1787695928.183768:0:38:0:(import.c:485:import_select_connection()) Process entered 02000000:00000010:1.0:1787695928.183769:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675800. 00000100:00080000:3.0:1787695928.183771:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3815 02000000:00000001:1.0:1787695928.183776:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:3.0:1787695928.183776:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:1.0:1787695928.183779:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695928.183782:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6680. 00000400:00000200:3.0:1787695928.183782:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695928.183785:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695928.183786:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695928.183788:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.183789:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.183795:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:3.0:1787695928.183795:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695928.183798:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695928.183799:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695928.183801:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1787695928.183804:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695928.183805:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695928.183808:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787695928.183809:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695928.183812:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000040:1.0:1787695928.183814:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5880 x1874531139546624/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1787695928.183814:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:3.0:1787695928.183815:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695928.183819:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000020:00000001:3.0:1787695928.183823:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000020:00000001:3.0:1787695928.183828:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:1.0:1787695928.183829:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695928.183831:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695928.183833:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695928.183834:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695928.183837:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695928.183837:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787695928.183841:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695928.183842:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00100000:1.0:1787695928.183846:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139546624:192.168.201.140@tcp:38:kworker.0 00000100:00000001:3.0:1787695928.183847:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695928.183851:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00000001:1.0:1787695928.183853:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695928.183855:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1787695928.183857:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000001:1.0:1787695928.183858:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695928.183862:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787695928.183863:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695928.183866:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547400. 02000000:00000001:3.0:1787695928.183866:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695928.183870:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695928.183871:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1787695928.183875:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787695928.183875:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695928.183878:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000010:1.0:1787695928.183880:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000008:00100000:3.0:1787695928.183881:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000010:1.0:1787695928.183884:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000008:00000001:3.0:1787695928.183885:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695928.183888:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695928.183889:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139546624, portal 10 00000100:00000001:1.0:1787695928.183892:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000010:3.0:1787695928.183892:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5500. 00000100:00000001:1.0:1787695928.183896:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150016 : -128555779401600 : ffff8b14466f5880) 00000020:00000040:3.0:1787695928.183897:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:3.0:1787695928.183900:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695928.183902:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695928.183904:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5880 x1874531139546624/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695928.183904:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695928.183906:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695928.183907:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695928.183910:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 00000100:00000001:1.0:1787695928.183915:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695928.183919:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695928.183920:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695928.183923:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695928.183926:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139546624, offset 0 00000100:00100000:3.0:1787695928.183927:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5500 x1874531139547136/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695928.183933:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695928.183941:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695928.183943:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1787695928.183946:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:3.0:1787695928.183946:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695928.183951:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1787695928.183954:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a777d68] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5500 x1874531139547136/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695928.183957:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695928.183962:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695928.183967:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695928.183973:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17080:17080:256:4294967295] 192.168.201.140@tcp LPNI seq info [17079:17079:8:4294967295] 00000100:00000001:3.0:1787695928.183974:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695928.183980:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695928.183981:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:3.0:1787695928.183985:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695928.183988:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695928.183991:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1787695928.183993:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:3.0:1787695928.183994:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695928.183999:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3815 00000800:00000200:1.0:1787695928.184002:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695928.184006:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000400:00000200:3.0:1787695928.184006:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695928.184014:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000800:00000200:1.0:1787695928.184018:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00080000:3.0:1787695928.184019:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1787695928.184024:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 6 secs later) 00000100:00000040:3.0:1787695928.184026:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695928.184029:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.1:1787695928.184031:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:3.0:1787695928.184034:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000200:1.1:1787695928.184035:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 6 secs later) 00000100:00000001:3.0:1787695928.184039:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000100:1.0:1787695928.184044:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000020:00000001:3.0:1787695928.184045:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000010:1.0:1787695928.184049:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000020:00000040:3.0:1787695928.184050:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000020:00000001:3.0:1787695928.184055:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:1.0:1787695928.184056:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000020:00000040:3.0:1787695928.184058:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000400:00000200:1.0:1787695928.184064:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000020:00000001:3.0:1787695928.184064:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000020:00000001:3.0:1787695928.184069:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000400:00000200:1.0:1787695928.184071:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695928.184076:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000040:3.0:1787695928.184077:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695928.184080:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787695928.184081:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1787695928.184088:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5880 x1874531139546624/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695928.184091:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695928.184095:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695928.184100:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00080000:3.0:1787695928.184106:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695928.184107:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787695928.184108:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695928.184111:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1787695928.184113:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5880 x1874531139546624/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695953 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695928.184114:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695928.184137:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695928.184140:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695928.184143:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695928.184149:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695928.184151:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695928.184155:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5f80. 00000020:00000040:3.0:1787695928.184161:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:3.0:1787695928.184168:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695928.184173:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.184174:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695928.184177:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787695928.184179:0:2402:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1787695928.184182:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695928.184186:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.184187:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695928.184193:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 00000100:00000040:1.0:1787695928.184194:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000400:00000200:1.0:1787695928.184216:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695928.184217:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.184218:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298489798 00000100:00000001:1.0:1787695928.184222:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298499897 00000100:00100000:3.0:1787695928.184224:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5f80 x1874531139547264/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695928.184226:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298489798 00000100:00000001:1.0:1787695928.184230:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298499897 00000100:00000001:1.0:1787695928.184235:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695928.184240:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695928.184240:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695928.184242:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787695928.184244:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695928.184248:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695928/real 1787695928] req@ffff8b14466f5880 x1874531139546624/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695928.184250:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006e7bde76] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5f80 x1874531139547264/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695928.184276:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787695928.184279:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:3.0:1787695928.184279:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:1.0:1787695928.184282:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:3.0:1787695928.184284:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000010:1.0:1787695928.184285:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000100:00000040:3.0:1787695928.184289:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695928.184290:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695928.184293:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695928.184298:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:3.0:1787695928.184299:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695928.184305:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3826) 00000100:00000200:1.0:1787695928.184307:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5880 x1874531139546624/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695928.184319:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5880 x1874531139546624/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.184326:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695928.184328:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.184330:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695928.184331:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695928.184335:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5880 x1874531139546624/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.184346:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695928.184349:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5880 x1874531139546624/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.184354:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695928.184356:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.184357:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695928.184358:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.184359:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695928.184360:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695928.184362:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695928.184364:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695928.184365:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695928.184367:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5880 x1874531139546624/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695928.184372:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139546624:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695928.184375:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695928.184377:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4a80 x1874531139546752/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.184382:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695928.184384:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695928.184385:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695928.184387:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695928.184388:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695928.184390:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139546752:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695928.184392:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695928.184409:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695928.184414:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695928.184416:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695928.184418:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787695928.184419:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695928.184439:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695928.184441:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695928.184443:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695928.184445:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139546752, portal 10 00000100:00000001:1.0:1787695928.184447:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695928.184448:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930146432 : -128555779405184 : ffff8b14466f4a80) 00000100:00000040:1.0:1787695928.184453:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4a80 x1874531139546752/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.184457:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695928.184459:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695928.184460:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695928.184462:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139546752, offset 0 00000400:00000200:1.0:1787695928.184465:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695928.184468:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695928.184471:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695928.184473:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695928.184475:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695928.184477:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695928.184479:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695928.184481:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17081:17081:256:4294967295] 192.168.201.140@tcp LPNI seq info [17080:17080:8:4294967295] 00000400:00000200:1.0:1787695928.184484:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695928.184489:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695928.184492:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695928.184494:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695928.184496:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695928.184499:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1787695928.184502:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695928.184504:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 6 secs later) 00000800:00000100:1.0:1787695928.184507:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695928.184511:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695928.184514:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695928.184517:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695928.184519:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695928.184520:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695928.184522:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695928.184524:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4a80 x1874531139546752/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.184530:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695928.184532:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4a80 x1874531139546752/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695953 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.184537:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.184538:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695928.184540:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695928.184543:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787695928.184545:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695928.184546:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298488647 00000100:00000001:1.0:1787695928.184548:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298499897 00000100:00000001:1.0:1787695928.184550:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298488647 00000100:00000001:1.0:1787695928.184551:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298499897 00000100:00000001:1.0:1787695928.184553:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695928.184554:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.184555:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695928.184558:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695928/real 1787695928] req@ffff8b14466f4a80 x1874531139546752/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695928.184563:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695928.184564:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695928.184565:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695928.184566:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695928.184568:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695928.184572:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695928.184575:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4a80 x1874531139546752/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695928.184580:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4a80 x1874531139546752/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.184584:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695928.184585:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.184587:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695928.184587:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695928.184590:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4a80 x1874531139546752/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.184594:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695928.184597:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4a80 x1874531139546752/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.184618:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695928.184620:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.184621:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695928.184621:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.184623:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695928.184624:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695928.184626:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695928.184628:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695928.184629:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695928.184632:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4a80 x1874531139546752/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695928.184637:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139546752:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695928.184639:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695928.184646:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4000 x1874531139546880/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.184651:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695928.184652:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695928.184654:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695928.184655:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695928.184656:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695928.184658:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139546880:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695928.184674:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695928.184677:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695928.184678:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695928.184680:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695928.184683:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545800. 02000000:00000001:1.0:1787695928.184685:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695928.184688:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695928.184690:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695928.184709:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695928.184712:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139546880, portal 4 00000100:00000001:1.0:1787695928.184713:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695928.184714:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930143744 : -128555779407872 : ffff8b14466f4000) 00000100:00000040:1.0:1787695928.184719:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4000 x1874531139546880/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.184724:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695928.184725:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695928.184727:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695928.184728:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139546880, offset 0 00000400:00000200:1.0:1787695928.184731:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695928.184734:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695928.184736:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695928.184738:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695928.184740:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695928.184742:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695928.184744:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695928.184750:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17082:17082:256:4294967295] 192.168.201.140@tcp LPNI seq info [17081:17081:8:4294967295] 00000400:00000200:1.0:1787695928.184753:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695928.184758:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695928.184762:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695928.184764:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695928.184766:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695928.184768:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1787695928.184771:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695928.184773:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 6 secs later) 00000800:00000100:1.0:1787695928.184779:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695928.184780:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695928.184782:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695928.184785:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695928.184787:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695928.184788:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695928.184790:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695928.184809:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4000 x1874531139546880/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.184814:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695928.184816:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4000 x1874531139546880/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695953 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.184820:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.184821:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695928.184823:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695928.184826:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695928.184827:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695928.184829:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298488677 00000100:00000001:1.0:1787695928.184830:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298499897 00000100:00000001:1.0:1787695928.184832:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298488677 00000100:00000001:1.0:1787695928.184833:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298499897 00000100:00000001:1.0:1787695928.184835:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695928.184839:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.184840:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695928.184843:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695928/real 1787695928] req@ffff8b14466f4000 x1874531139546880/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695928.184849:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695928.184850:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695928.184851:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695928.184852:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695928.184853:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695928.184855:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695928.184857:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4000 x1874531139546880/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695928.184861:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4000 x1874531139546880/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.184865:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695928.184866:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.184868:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695928.184869:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695928.184871:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4000 x1874531139546880/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.184875:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695928.184877:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4000 x1874531139546880/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.184881:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695928.184882:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.184883:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695928.184884:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.184885:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695928.184886:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695928.184888:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695928.184893:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695928.184894:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695928.184897:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4000 x1874531139546880/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695928.184902:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139546880:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695928.184904:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695928.184906:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4700 x1874531139547008/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.184910:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695928.184911:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695928.184913:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695928.184914:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695928.184915:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695928.184917:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139547008:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695928.184919:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695928.184920:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695928.184921:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695928.184922:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695928.184924:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546800. 02000000:00000001:1.0:1787695928.184925:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695928.184927:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695928.184929:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695928.184931:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695928.184933:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139547008, portal 4 00000100:00000001:1.0:1787695928.184934:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695928.184935:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930145536 : -128555779406080 : ffff8b14466f4700) 00000100:00000040:1.0:1787695928.184939:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4700 x1874531139547008/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.184943:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695928.184948:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695928.184949:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695928.184951:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139547008, offset 0 00000400:00000200:1.0:1787695928.184953:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695928.184956:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695928.184958:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695928.184960:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695928.184962:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695928.184980:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695928.184982:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695928.184984:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17083:17083:256:4294967295] 192.168.201.140@tcp LPNI seq info [17082:17082:8:4294967295] 00000400:00000200:1.0:1787695928.184987:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695928.184991:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695928.184994:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695928.184996:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695928.184999:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695928.185000:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1787695928.185003:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695928.185005:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 6 secs later) 00000800:00000100:1.0:1787695928.185007:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695928.185009:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695928.185011:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695928.185013:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695928.185015:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695928.185016:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695928.185018:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695928.185021:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4700 x1874531139547008/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185026:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695928.185028:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4700 x1874531139547008/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695953 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185049:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.185051:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695928.185059:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695928.185062:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695928.185066:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695928.185068:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298488349 00000100:00000001:1.0:1787695928.185071:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298499897 00000100:00000001:1.0:1787695928.185073:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298488349 00000100:00000001:1.0:1787695928.185076:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298499897 00000100:00000001:1.0:1787695928.185078:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695928.185080:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.185082:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695928.185087:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695928/real 1787695928] req@ffff8b14466f4700 x1874531139547008/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695928.185096:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695928.185097:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695928.185099:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695928.185102:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695928.185103:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695928.185105:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695928.185107:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4700 x1874531139547008/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695928.185113:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4700 x1874531139547008/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185120:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695928.185122:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.185124:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695928.185125:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695928.185128:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4700 x1874531139547008/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185136:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695928.185141:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4700 x1874531139547008/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185155:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695928.185158:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.185160:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695928.185161:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.185163:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695928.185164:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695928.185166:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695928.185168:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695928.185170:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695928.185173:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4700 x1874531139547008/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695928.185178:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139547008:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695928.185181:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.185182:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695928.185184:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5880 x1874531139546624/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185188:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695928.185190:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695928.185191:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547400. 02000000:00000001:1.0:1787695928.185193:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695928.185194:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695928.185195:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695928.185196:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695928.185198:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675400. 02000000:00000001:1.0:1787695928.185200:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695928.185201:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695928.185202:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5880. 00000100:00000001:1.0:1787695928.185204:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695928.185205:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.185209:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695928.185211:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4a80 x1874531139546752/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185216:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695928.185217:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695928.185218:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787695928.185220:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695928.185221:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695928.185222:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695928.185223:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695928.185224:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:1.0:1787695928.185227:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695928.185227:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695928.185228:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4a80. 00000100:00000001:1.0:1787695928.185230:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695928.185230:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.185231:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695928.185233:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4000 x1874531139546880/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185237:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695928.185238:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695928.185239:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545800. 02000000:00000001:1.0:1787695928.185240:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695928.185241:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695928.185242:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695928.185243:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695928.185244:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676000. 02000000:00000001:1.0:1787695928.185246:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695928.185246:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695928.185247:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4000. 00000100:00000001:1.0:1787695928.185248:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695928.185249:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.185250:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695928.185252:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4700 x1874531139547008/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185262:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695928.185263:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695928.185264:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546800. 02000000:00000001:1.0:1787695928.185266:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695928.185267:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695928.185268:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695928.185269:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695928.185270:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:1.0:1787695928.185271:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695928.185272:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695928.185272:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4700. 00000100:00000001:1.0:1787695928.185274:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695928.185274:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.185276:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.185278:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695928.185279:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.185281:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695928.185281:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695928.185282:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695928.185285:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5500 x1874531139547136/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185289:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695928.185291:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695928.185293:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695928.185294:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695928.185296:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695928.185299:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139547136:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695928.185302:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695928.185304:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695928.185306:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695928.185307:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695928.185310:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546800. 02000000:00000001:1.0:1787695928.185312:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695928.185321:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695928.185324:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695928.185328:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695928.185331:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139547136, portal 10 00000100:00000001:1.0:1787695928.185333:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695928.185334:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930149120 : -128555779402496 : ffff8b14466f5500) 00000100:00000040:1.0:1787695928.185341:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5500 x1874531139547136/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185347:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695928.185350:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695928.185352:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695928.185354:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139547136, offset 0 00000400:00000200:1.0:1787695928.185358:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695928.185362:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695928.185366:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695928.185369:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695928.185372:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695928.185374:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695928.185377:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695928.185381:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17084:17084:256:4294967295] 192.168.201.140@tcp LPNI seq info [17083:17083:8:4294967295] 00000400:00000200:1.0:1787695928.185386:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695928.185396:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695928.185401:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695928.185405:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695928.185410:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695928.185412:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1787695928.185416:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695928.185418:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 6 secs later) 00000800:00000100:1.0:1787695928.185421:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695928.185423:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695928.185448:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695928.185451:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695928.185453:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695928.185459:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695928.185461:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695928.185464:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5500 x1874531139547136/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185471:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695928.185473:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5500 x1874531139547136/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695953 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185478:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.185479:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695928.185482:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695928.185485:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695928.185488:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695928.185490:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298488536 00000100:00000001:1.0:1787695928.185509:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298499897 00000100:00000001:1.0:1787695928.185511:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298488536 00000100:00000001:1.0:1787695928.185514:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298499897 00000100:00000001:1.0:1787695928.185516:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695928.185518:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.185520:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695928.185524:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695928/real 1787695928] req@ffff8b14466f5500 x1874531139547136/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695928.185533:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695928.185535:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695928.185537:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695928.185540:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695928.185542:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695928.185544:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695928.185547:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5500 x1874531139547136/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695928.185555:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5500 x1874531139547136/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185568:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695928.185570:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.185572:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695928.185573:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695928.185577:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5500 x1874531139547136/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185583:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695928.185587:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5500 x1874531139547136/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185594:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695928.185597:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.185599:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695928.185600:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.185602:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695928.185604:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695928.185607:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695928.185611:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695928.185613:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695928.185618:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5500 x1874531139547136/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695928.185623:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139547136:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695928.185626:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695928.185628:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5f80 x1874531139547264/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185633:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695928.185634:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695928.185640:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695928.185641:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695928.185643:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695928.185645:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139547264:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695928.185647:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695928.185648:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695928.185650:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695928.185651:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695928.185653:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545800. 02000000:00000001:1.0:1787695928.185654:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695928.185657:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695928.185689:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695928.185692:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695928.185694:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139547264, portal 10 00000100:00000001:1.0:1787695928.185695:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695928.185696:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930151808 : -128555779399808 : ffff8b14466f5f80) 00000100:00000040:1.0:1787695928.185701:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5f80 x1874531139547264/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185705:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695928.185707:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695928.185708:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695928.185710:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139547264, offset 0 00000400:00000200:1.0:1787695928.185712:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695928.185715:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695928.185717:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695928.185719:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695928.185721:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695928.185723:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695928.185724:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695928.185727:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17085:17085:256:4294967295] 192.168.201.140@tcp LPNI seq info [17084:17084:8:4294967295] 00000400:00000200:1.0:1787695928.185729:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695928.185734:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695928.185742:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695928.185744:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695928.185747:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695928.185749:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 6 secs later) 00000800:00000200:1.1:1787695928.185752:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695928.185754:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 6 secs later) 00000800:00000100:1.0:1787695928.185756:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695928.185757:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695928.185759:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695928.185762:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695928.185764:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695928.185765:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695928.185767:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695928.185769:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5f80 x1874531139547264/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695953 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185775:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695928.185777:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5f80 x1874531139547264/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695953 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185781:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.185782:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695928.185784:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695928.185786:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695928.185789:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695928.185790:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298488514 00000100:00000001:1.0:1787695928.185792:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298499897 00000100:00000001:1.0:1787695928.185793:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298488514 00000100:00000001:1.0:1787695928.185795:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298499897 00000100:00000001:1.0:1787695928.185796:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695928.185805:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.185807:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695928.185812:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695928/real 1787695928] req@ffff8b14466f5f80 x1874531139547264/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695928.185822:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695928.185823:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695928.185824:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695928.185825:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695928.185826:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695928.185828:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695928.185830:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5f80 x1874531139547264/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695928.185836:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5f80 x1874531139547264/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185840:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695928.185841:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.185842:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695928.185843:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695928.185845:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5f80 x1874531139547264/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185849:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695928.185851:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5f80 x1874531139547264/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185855:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695928.185857:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.185858:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695928.185859:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.185861:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695928.185861:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695928.185863:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695928.185865:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695928.185866:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695928.185872:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5f80 x1874531139547264/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695928.185877:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139547264:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695928.185879:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.185881:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695928.185882:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5500 x1874531139547136/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185888:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695928.185890:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695928.185891:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546800. 02000000:00000001:1.0:1787695928.185894:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695928.185896:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695928.185897:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695928.185899:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695928.185901:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:1.0:1787695928.185903:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695928.185905:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695928.185906:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5500. 00000100:00000001:1.0:1787695928.185909:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695928.185910:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.185912:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695928.185915:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5f80 x1874531139547264/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695953 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695928.185923:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695928.185924:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695928.185926:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545800. 02000000:00000001:1.0:1787695928.185928:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695928.185929:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695928.185930:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695928.185933:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695928.185934:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:1.0:1787695928.185936:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695928.185942:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695928.185944:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5f80. 00000100:00000001:1.0:1787695928.185946:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695928.185947:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.185949:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695928.185952:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695928.185954:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695928.185956:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695928.185957:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695928.246420:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695928.246428:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695928.246429:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695928.246434:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695928.246437:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695928.246440:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695928.246441:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695928.246443:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695928.246445:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695928.886478:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695928.886485:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695928.886492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695928.886493:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695928.886497:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695928.886499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695928.886503:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695928.886504:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695928.886508:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695928.886509:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695928.886512:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695928.886514:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695928.886577:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695928.886581:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695928.886585:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695929.270484:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695929.270495:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695929.270497:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695929.270503:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695929.270507:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695929.270511:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695929.270513:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695929.270515:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695929.270518:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695929.910468:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695929.910473:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695929.910477:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695929.910478:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695929.910481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695929.910482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695929.910485:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695929.910485:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695929.910493:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695929.910494:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695929.910497:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695929.910497:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695929.910554:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695929.910557:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695929.910560:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695930.294460:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695930.294465:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695930.294467:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695930.294471:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695930.294474:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695930.294477:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695930.294479:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695930.294481:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695930.294483:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695930.934522:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695930.934545:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695930.934553:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695930.934554:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695930.934559:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695930.934585:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695930.934591:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695930.934592:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695930.934595:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695930.934596:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695930.934599:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695930.934600:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695930.934700:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695930.934706:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695930.934710:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695931.318473:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695931.318481:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695931.318484:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695931.318489:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695931.318514:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695931.318518:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695931.318521:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695931.318523:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695931.318526:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695931.958483:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695931.958494:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695931.958502:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695931.958504:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695931.958509:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695931.958511:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695931.958514:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695931.958515:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695931.958518:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695931.958522:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695931.958525:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695931.958531:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695931.958533:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695931.958535:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695931.958538:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695931.958540:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695931.958543:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695931.958544:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695931.958547:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695931.958548:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695931.958631:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695931.958635:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695931.958638:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695932.342456:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695932.342461:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695932.342463:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695932.342467:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695932.342470:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695932.342473:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695932.342475:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695932.342477:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695932.342479:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695932.982509:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695932.982517:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695932.982522:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695932.982523:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695932.982526:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695932.982528:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695932.982531:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695932.982532:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695932.982535:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695932.982536:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695932.982539:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695932.982559:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695932.982642:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695932.982648:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695932.982651:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787695933.302476:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695933.302487:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787695933.302490:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695933.302494:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695933.302498:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695933.302507:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3821 00000400:00000200:3.0:1787695933.302517:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695933.302523:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695933.302527:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695933.302530:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695933.302534:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695933.302544:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695933.302547:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695933.302551:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695933.302554:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:3.0:1787695933.302556:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695933.302560:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787695933.302563:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787695933.302566:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787695933.302596:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695933.302599:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695933.302603:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695933.302606:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695933.302609:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787695933.302613:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695933.302616:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695933.302619:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695933.302624:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695933.302628:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695933.302631:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787695933.302633:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695933.302640:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4380. 00000020:00000040:3.0:1787695933.302644:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787695933.302649:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695933.302651:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695933.302654:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695933.302656:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695933.302661:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695933.302669:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674800. 00000100:00000001:3.0:1787695933.302676:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695933.302686:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4380 x1874531139547392/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695933.302698:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695933.302702:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695933.302707:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d2982c15] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4380 x1874531139547392/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695933.303070:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695933.303073:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695933.303077:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695933.303078:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695933.303086:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695933.303089:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695933.303093:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3821 00000400:00000200:3.0:1787695933.303098:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695933.303102:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695933.303105:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695933.303107:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695933.303110:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695933.303112:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695933.303115:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695933.303119:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695933.303120:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:3.0:1787695933.303123:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695933.303126:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:3.0:1787695933.303129:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:3.0:1787695933.303131:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:3.0:1787695933.303134:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695933.303136:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695933.303139:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695933.303141:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695933.303143:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695933.303149:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00000001:1.0:1787695933.303154:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1787695933.303157:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695933.303161:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787695933.303177:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695933.303180:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4380 x1874531139547392/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695933.303183:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695933.303192:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695933.303200:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695933.303206:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695933.303206:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695933.303216:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787695933.303219:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:3.0:1787695933.303223:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787695933.303226:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:3.0:1787695933.303233:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695933.303236:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695933.303239:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695933.303245:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139547392:192.168.201.140@tcp:250:kworker.0 00000100:00000001:1.0:1787695933.303256:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000010:3.0:1787695933.303257:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5180. 00000020:00000040:3.0:1787695933.303262:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:1.0:1787695933.303264:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787695933.303266:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695933.303270:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695933.303271:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695933.303276:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695933.303279:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787695933.303280:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695933.303289:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695933.303296:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545800. 02000000:00000010:3.0:1787695933.303296:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 02000000:00000001:1.0:1787695933.303305:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695933.303308:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695933.303322:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00100000:3.0:1787695933.303325:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5180 x1874531139547520/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1787695933.303330:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695933.303344:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000001:3.0:1787695933.303344:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695933.303350:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695933.303359:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139547392, portal 25 00000100:00000040:3.0:1787695933.303360:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a115abf4] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5180 x1874531139547520/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.303419:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695933.303423:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930144640 : -128555779406976 : ffff8b14466f4380) 00000100:00000040:1.0:1787695933.303438:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4380 x1874531139547392/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.303450:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787695933.303450:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695933.303460:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1787695933.303461:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00080000:3.0:1787695933.303469:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000010:1.0:1787695933.303471:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000001:3.0:1787695933.303472:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695933.303477:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000200:1.0:1787695933.303481:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139547392, offset 0 00000100:00000001:3.0:1787695933.303483:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695933.303493:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3821 00000400:00000200:1.0:1787695933.303502:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:3.0:1787695933.303509:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695933.303516:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695933.303521:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1787695933.303522:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695933.303539:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695933.303557:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695933.303563:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695933.303568:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695933.303571:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695933.303594:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17086:17086:256:4294967295] 192.168.201.140@tcp LPNI seq info [17085:17085:8:4294967295] 00000400:00000200:1.0:1787695933.303601:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695933.303610:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695933.303619:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695933.303623:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000040:3.0:1787695933.303634:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000200:1.0:1787695933.303635:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695933.303644:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000010:1.0:1787695933.303649:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000020:00000001:3.0:1787695933.303651:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000200:1.0:1787695933.303661:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000020:00000040:3.0:1787695933.303663:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000800:00000200:1.0:1787695933.303669:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 1 secs later) 00000020:00000001:3.0:1787695933.303669:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695933.303674:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:3.0:1787695933.303680:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000800:00000200:1.1:1787695933.303682:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000020:00000001:3.0:1787695933.303687:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000800:00000200:1.1:1787695933.303691:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 1 secs later) 00000100:00000040:3.0:1787695933.303699:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000800:00000100:1.0:1787695933.303706:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695933.303709:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:1.0:1787695933.303712:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000100:00000040:3.0:1787695933.303719:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695933.303722:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1787695933.303724:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695933.303730:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000400:00000200:1.0:1787695933.303735:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00080000:3.0:1787695933.303741:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:1.0:1787695933.303745:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000100:00000001:3.0:1787695933.303747:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695933.303752:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695933.303756:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1787695933.303760:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 02000000:00000001:3.0:1787695933.303761:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695933.303768:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695933.303771:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787695933.303772:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000008:00000001:3.0:1787695933.303778:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695933.303785:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4380 x1874531139547392/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695933.303786:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695933.303796:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6d80. 00000020:00000040:3.0:1787695933.303802:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695933.303805:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695933.303806:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695933.303811:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695933.303812:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695933.303816:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695933.303829:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 00000100:00000001:1.0:1787695933.303830:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695933.303840:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4380 x1874531139547392/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695958 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695933.303841:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695933.303859:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6d80 x1874531139547648/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695933.303874:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695933.303890:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695933.303891:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787695933.303899:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.303905:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695933.303911:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c3be57b] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6d80 x1874531139547648/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695933.303921:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695933.303931:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298505017 00000100:00000001:1.0:1787695933.303941:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298499897 00000100:00000001:3.0:1787695933.303943:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695933.303949:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695933.303952:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298505017 00000100:00080000:3.0:1787695933.303952:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787695933.303956:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695933.303957:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695933.303961:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695933.303963:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3821 00000400:00000200:3.0:1787695933.303971:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695933.303976:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298499897 00000400:00000200:3.0:1787695933.303977:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695933.303989:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695933.303991:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1787695933.303996:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695933.304005:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695933.304005:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695933.304012:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695933.304015:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787695933.304016:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695933.304023:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695933.304025:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:3.0:1787695933.304027:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695933.304033:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000100:00100000:1.0:1787695933.304034:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695933/real 1787695933] req@ffff8b14466f4380 x1874531139547392/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695933.304042:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:3.0:1787695933.304051:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:3.0:1787695933.304066:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695933.304076:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695933.304082:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695933.304089:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000001:1.0:1787695933.304092:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000040:3.0:1787695933.304094:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000400:00000001:1.0:1787695933.304097:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695933.304102:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000100:00080000:3.0:1787695933.304102:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:1.0:1787695933.304109:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000100:00000001:3.0:1787695933.304110:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695933.304114:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695933.304119:0:2402:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1787695933.304122:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:1.0:1787695933.304130:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4380 x1874531139547392/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695933.304133:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695933.304144:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695933.304153:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695933.304159:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000200:1.0:1787695933.304160:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4380 x1874531139547392/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1787695933.304175:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.304186:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787695933.304186:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.304190:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.304196:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000010:3.0:1787695933.304197:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5f80. 00000100:00000001:1.0:1787695933.304199:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1787695933.304209:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000100:00100000:1.0:1787695933.304212:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4380 x1874531139547392/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.304223:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787695933.304231:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695933.304233:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695933.304234:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4380 x1874531139547392/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695933.304243:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695933.304248:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695933.304254:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695933.304264:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 00000100:00000001:1.0:1787695933.304265:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787695933.304272:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.304273:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.304277:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695933.304283:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787695933.304283:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5f80 x1874531139547776/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695933.304292:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695933.304302:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787695933.304305:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695933.304309:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695933.304310:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:3.0:1787695933.304319:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006e7bde76] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5f80 x1874531139547776/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695933.304325:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695933.304335:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787695933.304338:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695933.304342:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1787695933.304349:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4380 x1874531139547392/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695933.304350:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1787695933.304355:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695933.304359:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695933.304387:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695933.304391:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3821 00000400:00000200:3.0:1787695933.304408:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00100000:1.0:1787695933.304423:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139547392:192.168.201.140@tcp:250:kworker.0 00000400:00000200:3.0:1787695933.304423:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695933.304433:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695933.304440:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.304448:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1787695933.304450:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695933.304455:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695933.304458:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4380 x1874531139547392/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695933.304459:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695933.304468:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695933.304477:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695933.304481:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:3.0:1787695933.304487:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695933.304491:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000100:00000001:1.0:1787695933.304493:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000020:00000001:3.0:1787695933.304498:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 02000000:00000001:1.0:1787695933.304501:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:3.0:1787695933.304503:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 02000000:00000010:1.0:1787695933.304508:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545800. 00000100:00000040:3.0:1787695933.304509:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695933.304518:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787695933.304523:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:3.0:1787695933.304523:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695933.304530:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695933.304532:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1787695933.304533:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000020:00000040:1.0:1787695933.304536:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000100:00080000:3.0:1787695933.304541:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000020:00000001:1.0:1787695933.304545:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:3.0:1787695933.304546:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695933.304552:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:1.0:1787695933.304553:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674800. 02000000:00000001:3.0:1787695933.304558:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695933.304562:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787695933.304564:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695933.304566:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787695933.304569:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000010:1.0:1787695933.304571:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4380. 00000008:00100000:3.0:1787695933.304586:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787695933.304591:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000008:00000001:3.0:1787695933.304593:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.304595:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695933.304599:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.304601:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:3.0:1787695933.304608:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5500. 00000100:00000001:1.0:1787695933.304613:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:3.0:1787695933.304614:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 00000100:00000001:1.0:1787695933.304618:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695933.304620:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695933.304625:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.304626:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695933.304631:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695933.304633:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787695933.304635:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787695933.304638:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1787695933.304645:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5180 x1874531139547520/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695933.304646:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695933.304661:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676000. 00000100:00000001:3.0:1787695933.304679:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.304696:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1787695933.304698:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5500 x1874531139547904/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695933.304700:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695933.304708:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695933.304715:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695933.304722:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695933.304728:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1787695933.304732:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139547520:192.168.201.140@tcp:38:kworker.0 00000100:00000001:3.0:1787695933.304734:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.304743:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695933.304748:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1787695933.304750:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a777d68] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5500 x1874531139547904/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695933.304751:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695933.304758:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695933.304764:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545800. 02000000:00000001:1.0:1787695933.304770:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695933.304777:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695933.304787:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:3.0:1787695933.304788:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:1.0:1787695933.304799:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000100:00080000:3.0:1787695933.304800:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695933.304804:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695933.304809:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:1.0:1787695933.304811:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000001:3.0:1787695933.304814:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000200:1.0:1787695933.304821:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139547520, portal 10 00000100:00080000:3.0:1787695933.304821:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3821 00000100:00000001:1.0:1787695933.304833:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695933.304841:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930148224 : -128555779403392 : ffff8b14466f5180) 00000400:00000200:3.0:1787695933.304842:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695933.304852:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695933.304857:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695933.304866:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5180 x1874531139547520/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695933.304866:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695933.304879:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695933.304889:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1787695933.304891:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695933.304899:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695933.304901:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000010:1.0:1787695933.304904:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000020:00000001:3.0:1787695933.304909:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000200:1.0:1787695933.304911:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139547520, offset 0 00000020:00000040:3.0:1787695933.304914:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000400:00000200:1.0:1787695933.304922:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000020:00000001:3.0:1787695933.304923:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695933.304931:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000400:00000200:1.0:1787695933.304939:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:3.0:1787695933.304941:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000200:1.0:1787695933.304950:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000020:00000001:3.0:1787695933.304950:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000200:1.0:1787695933.304960:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1787695933.304966:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695933.304970:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:3.0:1787695933.304975:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695933.304978:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695933.304982:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000040:3.0:1787695933.304989:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695933.304996:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695933.305002:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17087:17087:256:4294967295] 192.168.201.140@tcp LPNI seq info [17086:17086:8:4294967295] 00000020:00000040:3.0:1787695933.305005:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:3.0:1787695933.305017:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:1.0:1787695933.305019:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695933.305026:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695933.305031:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1787695933.305038:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 02000000:00000001:3.0:1787695933.305039:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:1.0:1787695933.305051:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695933.305052:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1787695933.305058:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000100:00000001:3.0:1787695933.305059:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695933.305068:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000800:00000200:1.0:1787695933.305070:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000008:00000001:3.0:1787695933.305078:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695933.305080:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 1 secs later) 00000100:00000001:3.0:1787695933.305084:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1787695933.305092:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000010:3.0:1787695933.305093:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4700. 00000800:00000200:1.1:1787695933.305099:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 1 secs later) 00000020:00000040:3.0:1787695933.305101:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000800:00000100:1.0:1787695933.305112:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695933.305112:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000010:1.0:1787695933.305119:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695933.305123:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695933.305127:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695933.305130:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 02000000:00000001:3.0:1787695933.305131:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695933.305134:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695933.305140:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787695933.305140:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695933.305146:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:1.0:1787695933.305149:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5180 x1874531139547520/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695933.305151:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695933.305165:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 00000100:00000001:1.0:1787695933.305173:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787695933.305174:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695933.305183:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5180 x1874531139547520/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695958 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695933.305194:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4700 x1874531139548032/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695933.305199:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.305205:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695933.305210:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695933.305219:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:1.0:1787695933.305221:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000100:00000001:3.0:1787695933.305227:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695933.305235:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695933.305243:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298494919 00000100:00000040:3.0:1787695933.305243:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008818c2cc] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4700 x1874531139548032/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.305252:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298505017 00000100:00000001:1.0:1787695933.305258:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298494919 00000100:00000001:1.0:1787695933.305265:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298505017 00000100:00000001:1.0:1787695933.305272:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695933.305277:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.305285:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695933.305288:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695933.305291:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1787695933.305296:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787695933.305302:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695933/real 1787695933] req@ffff8b14466f5180 x1874531139547520/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695933.305302:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695933.305314:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:1.0:1787695933.305318:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787695933.305322:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000001:1.0:1787695933.305323:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695933.305328:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000100:00080000:3.0:1787695933.305330:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3821 00000400:00000200:1.0:1787695933.305335:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695933.305341:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000400:00000200:3.0:1787695933.305342:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695933.305348:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:3.0:1787695933.305353:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000200:1.0:1787695933.305359:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5180 x1874531139547520/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695933.305361:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000200:1.0:1787695933.305399:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5180 x1874531139547520/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695933.305402:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695933.305412:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695933.305421:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695933.305423:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787695933.305428:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695933.305429:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787695933.305432:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695933.305437:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:3.0:1787695933.305438:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000100:00000001:1.0:1787695933.305446:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787695933.305446:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695933.305452:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:3.0:1787695933.305458:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00100000:1.0:1787695933.305460:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5180 x1874531139547520/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695933.305463:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:3.0:1787695933.305474:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695933.305484:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695933.305494:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695933.305496:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695933.305498:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695933.305504:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00000040:1.0:1787695933.305510:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5180 x1874531139547520/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695933.305512:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695933.305521:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695933.305529:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695933.305536:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695933.305539:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1787695933.305540:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.305547:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695933.305548:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695933.305556:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00100000:3.0:1787695933.305556:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787695933.305559:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.305566:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000008:00000001:3.0:1787695933.305567:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.305570:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787695933.305585:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695933.305589:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000010:3.0:1787695933.305600:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4000. 00000100:00080000:1.0:1787695933.305608:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000020:00000040:3.0:1787695933.305609:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000001:1.0:1787695933.305614:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1787695933.305614:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695933.305620:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695933.305625:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:1.0:1787695933.305631:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5180 x1874531139547520/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695933.305631:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695933.305635:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695933.305647:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675400. 00000100:00000001:3.0:1787695933.305657:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695933.305670:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139547520:192.168.201.140@tcp:38:kworker.0 00000100:00100000:3.0:1787695933.305671:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4000 x1874531139548160/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695933.305690:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787695933.305690:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695933.305699:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695933.305706:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5a8f356] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4000 x1874531139548160/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695933.305708:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6d80 x1874531139547648/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695933.305741:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695933.305747:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695933.305750:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:3.0:1787695933.305751:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1787695933.305759:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695933.305759:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787695933.305770:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 02000000:00000001:1.0:1787695933.305771:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695933.305777:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1787695933.305778:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3831) 02000000:00000001:1.0:1787695933.305788:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695933.305796:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139547648:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695933.305801:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695933.305804:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695933.305806:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695933.305808:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695933.305813:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546800. 02000000:00000001:1.0:1787695933.305816:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695933.305821:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695933.305825:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695933.305828:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695933.305832:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139547648, portal 10 00000100:00000001:1.0:1787695933.305834:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695933.305836:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930155392 : -128555779396224 : ffff8b14466f6d80) 00000100:00000040:1.0:1787695933.305843:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6d80 x1874531139547648/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.305851:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695933.305861:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695933.305864:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695933.305867:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139547648, offset 0 00000400:00000200:1.0:1787695933.305872:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695933.305876:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695933.305881:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695933.305884:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695933.305887:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695933.305890:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695933.305893:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695933.305897:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17088:17088:256:4294967295] 192.168.201.140@tcp LPNI seq info [17087:17087:8:4294967295] 00000400:00000200:1.0:1787695933.305911:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695933.305923:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695933.305928:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695933.305932:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695933.305937:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695933.305940:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1787695933.305946:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695933.305949:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1787695933.305954:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695933.305956:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695933.305959:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695933.305962:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695933.305965:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695933.305967:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695933.305969:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695933.305973:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6d80 x1874531139547648/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.305981:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695933.305985:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6d80 x1874531139547648/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695958 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.305992:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.305994:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695933.306007:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695933.306011:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787695933.306015:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695933.306018:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298493768 00000100:00000001:1.0:1787695933.306021:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298505017 00000100:00000001:1.0:1787695933.306024:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298493768 00000100:00000001:1.0:1787695933.306027:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298505017 00000100:00000001:1.0:1787695933.306030:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695933.306032:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.306035:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695933.306040:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695933/real 1787695933] req@ffff8b14466f6d80 x1874531139547648/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695933.306049:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695933.306050:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695933.306053:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695933.306055:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695933.306057:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695933.306060:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695933.306063:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6d80 x1874531139547648/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695933.306073:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6d80 x1874531139547648/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.306080:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695933.306082:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.306083:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695933.306085:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695933.306089:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6d80 x1874531139547648/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.306097:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695933.306102:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6d80 x1874531139547648/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.306116:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695933.306119:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.306121:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695933.306122:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.306123:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695933.306125:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695933.306127:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695933.306130:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695933.306131:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695933.306134:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6d80 x1874531139547648/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695933.306140:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139547648:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695933.306143:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695933.306145:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5f80 x1874531139547776/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.306150:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695933.306151:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695933.306153:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695933.306155:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695933.306156:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695933.306158:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139547776:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695933.306160:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695933.306162:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695933.306163:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695933.306164:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695933.306168:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787695933.306170:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695933.306174:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695933.306182:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695933.306186:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695933.306189:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139547776, portal 4 00000100:00000001:1.0:1787695933.306191:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695933.306193:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930151808 : -128555779399808 : ffff8b14466f5f80) 00000100:00000040:1.0:1787695933.306200:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5f80 x1874531139547776/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.306207:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695933.306210:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695933.306213:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695933.306216:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139547776, offset 0 00000400:00000200:1.0:1787695933.306220:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695933.306225:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695933.306228:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695933.306232:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695933.306235:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695933.306237:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695933.306240:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695933.306243:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17089:17089:256:4294967295] 192.168.201.140@tcp LPNI seq info [17088:17088:8:4294967295] 00000400:00000200:1.0:1787695933.306247:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695933.306254:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695933.306259:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695933.306262:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695933.306266:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695933.306269:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1787695933.306274:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695933.306277:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1787695933.306281:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695933.306283:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695933.306287:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695933.306291:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695933.306294:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695933.306296:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695933.306304:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695933.306309:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5f80 x1874531139547776/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.306318:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695933.306321:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5f80 x1874531139547776/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695958 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.306329:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.306331:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695933.306335:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695933.306338:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695933.306342:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695933.306344:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298493798 00000100:00000001:1.0:1787695933.306347:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298505017 00000100:00000001:1.0:1787695933.306350:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298493798 00000100:00000001:1.0:1787695933.306352:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298505017 00000100:00000001:1.0:1787695933.306355:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695933.306357:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.306359:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695933.306384:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695933/real 1787695933] req@ffff8b14466f5f80 x1874531139547776/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695933.306393:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695933.306396:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695933.306400:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695933.306404:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695933.306407:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695933.306410:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695933.306414:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5f80 x1874531139547776/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695933.306425:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5f80 x1874531139547776/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.306444:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695933.306446:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.306448:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695933.306450:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695933.306455:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5f80 x1874531139547776/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.306463:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695933.306469:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5f80 x1874531139547776/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.306478:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695933.306481:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.306482:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695933.306484:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.306486:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695933.306489:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695933.306493:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695933.306497:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695933.306501:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695933.306508:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5f80 x1874531139547776/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695933.306517:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139547776:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695933.306521:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.306524:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695933.306527:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5180 x1874531139547520/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.306534:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695933.306536:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695933.306542:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545800. 02000000:00000001:1.0:1787695933.306545:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695933.306547:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695933.306549:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695933.306552:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695933.306554:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:1.0:1787695933.306557:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695933.306559:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695933.306561:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5180. 00000100:00000001:1.0:1787695933.306564:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695933.306565:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.306567:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695933.306571:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6d80 x1874531139547648/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.306594:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695933.306596:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695933.306598:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546800. 02000000:00000001:1.0:1787695933.306602:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695933.306604:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695933.306606:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695933.306608:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695933.306610:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:1.0:1787695933.306614:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695933.306615:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695933.306617:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6d80. 00000100:00000001:1.0:1787695933.306620:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695933.306621:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.306623:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695933.306627:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5f80 x1874531139547776/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.306634:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695933.306636:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695933.306637:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787695933.306640:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695933.306641:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695933.306643:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695933.306649:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695933.306651:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:1.0:1787695933.306656:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695933.306658:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695933.306660:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5f80. 00000100:00000001:1.0:1787695933.306662:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695933.306664:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.306666:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.306671:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695933.306672:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.306675:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695933.306677:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695933.306679:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695933.306683:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5500 x1874531139547904/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.306692:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695933.306693:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695933.306696:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695933.306698:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695933.306701:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695933.306704:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139547904:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695933.306708:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695933.306710:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695933.306712:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695933.306714:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695933.306716:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787695933.306719:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695933.306723:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695933.306725:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695933.306729:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695933.306733:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139547904, portal 4 00000100:00000001:1.0:1787695933.306735:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695933.306736:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930149120 : -128555779402496 : ffff8b14466f5500) 00000100:00000040:1.0:1787695933.306743:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5500 x1874531139547904/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.306763:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695933.306766:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695933.306768:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695933.306772:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139547904, offset 0 00000400:00000200:1.0:1787695933.306776:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695933.306781:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695933.306785:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695933.306788:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695933.306791:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695933.306794:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695933.306796:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695933.306800:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17090:17090:256:4294967295] 192.168.201.140@tcp LPNI seq info [17089:17089:8:4294967295] 00000400:00000200:1.0:1787695933.306805:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695933.306812:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695933.306817:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695933.306820:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695933.306825:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695933.306828:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1787695933.306833:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695933.306836:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1787695933.306842:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695933.306844:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695933.306847:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695933.306851:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695933.306854:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695933.306856:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695933.306859:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695933.306863:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5500 x1874531139547904/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.306872:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695933.306876:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5500 x1874531139547904/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695958 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.306895:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.306896:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695933.306900:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695933.306904:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695933.306908:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695933.306910:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298493471 00000100:00000001:1.0:1787695933.306913:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298505017 00000100:00000001:1.0:1787695933.306915:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298493471 00000100:00000001:1.0:1787695933.306918:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298505017 00000100:00000001:1.0:1787695933.306920:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695933.306922:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.306924:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695933.306928:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695933/real 1787695933] req@ffff8b14466f5500 x1874531139547904/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695933.306937:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695933.306938:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695933.306940:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695933.306942:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695933.306944:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695933.306947:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695933.306950:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5500 x1874531139547904/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695933.306960:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5500 x1874531139547904/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.306967:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695933.306968:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.306970:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695933.306971:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695933.306975:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5500 x1874531139547904/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.306987:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695933.306992:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5500 x1874531139547904/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.307000:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695933.307002:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.307004:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695933.307005:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.307008:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695933.307009:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695933.307012:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695933.307016:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695933.307018:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695933.307023:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5500 x1874531139547904/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695933.307032:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139547904:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695933.307036:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695933.307040:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4700 x1874531139548032/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.307049:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695933.307050:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695933.307053:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695933.307055:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695933.307057:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695933.307060:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139548032:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695933.307064:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695933.307066:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695933.307071:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695933.307073:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695933.307076:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546800. 02000000:00000001:1.0:1787695933.307078:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695933.307082:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695933.307085:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695933.307088:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695933.307091:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139548032, portal 10 00000100:00000001:1.0:1787695933.307093:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695933.307095:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930145536 : -128555779406080 : ffff8b14466f4700) 00000100:00000040:1.0:1787695933.307102:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4700 x1874531139548032/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.307110:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695933.307113:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695933.307115:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695933.307118:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139548032, offset 0 00000400:00000200:1.0:1787695933.307122:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695933.307125:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695933.307129:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695933.307132:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695933.307135:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695933.307138:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695933.307140:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695933.307144:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17091:17091:256:4294967295] 192.168.201.140@tcp LPNI seq info [17090:17090:8:4294967295] 00000400:00000200:1.0:1787695933.307149:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695933.307156:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695933.307161:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695933.307164:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695933.307169:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695933.307172:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1787695933.307177:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695933.307180:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1787695933.307184:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695933.307190:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695933.307193:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695933.307198:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695933.307201:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695933.307203:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695933.307206:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695933.307210:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4700 x1874531139548032/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.307219:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695933.307223:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4700 x1874531139548032/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695958 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.307230:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.307232:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695933.307235:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695933.307239:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695933.307243:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695933.307245:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298493657 00000100:00000001:1.0:1787695933.307248:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298505017 00000100:00000001:1.0:1787695933.307250:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298493657 00000100:00000001:1.0:1787695933.307253:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298505017 00000100:00000001:1.0:1787695933.307255:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695933.307258:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.307260:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695933.307264:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695933/real 1787695933] req@ffff8b14466f4700 x1874531139548032/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695933.307271:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695933.307273:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695933.307275:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695933.307277:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695933.307279:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695933.307285:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695933.307289:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4700 x1874531139548032/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695933.307299:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4700 x1874531139548032/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.307307:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695933.307308:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.307311:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695933.307312:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695933.307316:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4700 x1874531139548032/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.307323:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695933.307327:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4700 x1874531139548032/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.307334:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695933.307337:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.307338:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695933.307340:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.307342:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695933.307343:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695933.307346:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695933.307350:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695933.307352:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695933.307357:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4700 x1874531139548032/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695933.307399:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139548032:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695933.307404:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695933.307413:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4000 x1874531139548160/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.307421:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695933.307423:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695933.307426:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695933.307428:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695933.307431:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695933.307434:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139548160:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695933.307438:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695933.307440:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695933.307443:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695933.307445:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695933.307448:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545800. 02000000:00000001:1.0:1787695933.307450:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695933.307454:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695933.307457:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695933.307460:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000200:1.0:1787695933.307464:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139548160, portal 10 00000100:00000001:1.0:1787695933.307466:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695933.307468:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930143744 : -128555779407872 : ffff8b14466f4000) 00000100:00000040:1.0:1787695933.307474:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4000 x1874531139548160/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.307482:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695933.307485:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695933.307487:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000100:00000200:1.0:1787695933.307490:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139548160, offset 0 00000400:00000200:1.0:1787695933.307494:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695933.307498:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695933.307502:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695933.307505:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695933.307509:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695933.307511:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695933.307519:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695933.307523:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17092:17092:256:4294967295] 192.168.201.140@tcp LPNI seq info [17091:17091:8:4294967295] 00000400:00000200:1.0:1787695933.307528:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695933.307536:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695933.307541:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695933.307544:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000800:00000200:1.0:1787695933.307548:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695933.307551:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 1 secs later) 00000800:00000200:1.1:1787695933.307556:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695933.307559:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 1 secs later) 00000800:00000100:1.0:1787695933.307563:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695933.307566:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:1.0:1787695933.307569:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695933.307585:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695933.307589:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:1.0:1787695933.307591:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:1.0:1787695933.307594:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695933.307599:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4000 x1874531139548160/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695958 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.307607:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695933.307611:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4000 x1874531139548160/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695958 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.307619:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.307621:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695933.307624:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695933.307627:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695933.307631:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695933.307633:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298493636 00000100:00000001:1.0:1787695933.307636:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298505017 00000100:00000001:1.0:1787695933.307639:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298493636 00000100:00000001:1.0:1787695933.307641:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298505017 00000100:00000001:1.0:1787695933.307648:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695933.307650:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.307653:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695933.307657:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695933/real 1787695933] req@ffff8b14466f4000 x1874531139548160/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695933.307667:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695933.307668:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695933.307670:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695933.307673:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695933.307675:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695933.307677:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695933.307681:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4000 x1874531139548160/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695933.307691:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4000 x1874531139548160/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.307698:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695933.307699:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.307701:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695933.307702:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695933.307706:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4000 x1874531139548160/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.307713:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695933.307718:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4000 x1874531139548160/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.307726:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695933.307729:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.307731:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695933.307732:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.307734:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695933.307735:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695933.307742:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695933.307746:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695933.307748:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695933.307753:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4000 x1874531139548160/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695933.307762:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139548160:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695933.307766:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.307768:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695933.307771:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5500 x1874531139547904/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.307778:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695933.307780:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695933.307782:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787695933.307785:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695933.307787:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695933.307788:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695933.307791:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695933.307793:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676000. 02000000:00000001:1.0:1787695933.307796:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695933.307797:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695933.307799:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5500. 00000100:00000001:1.0:1787695933.307801:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695933.307803:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.307805:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695933.307808:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4700 x1874531139548032/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.307814:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695933.307816:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695933.307817:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546800. 02000000:00000001:1.0:1787695933.307820:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695933.307821:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695933.307828:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695933.307830:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695933.307832:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:1.0:1787695933.307834:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695933.307836:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695933.307837:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4700. 00000100:00000001:1.0:1787695933.307839:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695933.307841:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.307842:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695933.307846:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4000 x1874531139548160/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695958 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695933.307853:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695933.307854:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695933.307856:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545800. 02000000:00000001:1.0:1787695933.307858:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695933.307859:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695933.307861:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695933.307863:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695933.307864:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675400. 02000000:00000001:1.0:1787695933.307867:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695933.307869:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695933.307870:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4000. 00000100:00000001:1.0:1787695933.307872:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695933.307874:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.307876:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695933.307880:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695933.307881:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695933.307883:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695933.307885:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695933.366443:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695933.366450:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695933.366451:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695933.366456:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695933.366459:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695933.366462:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695933.366464:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695933.366466:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695933.366468:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695934.006472:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695934.006482:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695934.006484:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695934.006488:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695934.006494:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695934.006496:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695934.006498:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695934.006500:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695934.006526:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695934.006527:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695934.006529:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695934.006530:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695934.006531:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695934.006532:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695934.006534:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695934.006535:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695934.006538:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695934.006539:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695934.006542:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695934.006543:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695934.006545:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695934.006546:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695934.006548:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695934.006549:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695934.006631:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695934.006642:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695934.006645:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695934.390442:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695934.390449:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695934.390451:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695934.390455:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695934.390458:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695934.390460:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695934.390462:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695934.390465:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695934.390467:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695935.030520:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695935.030528:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695935.030534:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695935.030535:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695935.030538:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695935.030539:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695935.030557:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695935.030558:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695935.030561:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695935.030562:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695935.030565:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695935.030566:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695935.030667:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695935.030675:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695935.030680:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695935.414450:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695935.414456:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695935.414457:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695935.414462:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695935.414464:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695935.414467:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695935.414470:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695935.414472:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695935.414474:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695936.054478:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695936.054486:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695936.054492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695936.054493:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695936.054496:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695936.054498:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695936.054501:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695936.054501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695936.054504:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695936.054505:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695936.054578:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695936.054579:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695936.054669:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695936.054676:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695936.054681:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695936.438466:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695936.438472:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695936.438474:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695936.438478:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695936.438481:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695936.438484:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695936.438486:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695936.438488:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695936.438491:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695937.078550:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695937.078558:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695937.078564:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695937.078565:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695937.078568:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695937.078571:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695937.078574:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695937.078580:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695937.078582:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695937.078584:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695937.078587:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695937.078589:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695937.078591:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695937.078593:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695937.078596:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695937.078597:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695937.078598:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695937.078599:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695937.078600:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695937.078600:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695937.078601:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695937.078602:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695937.078603:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695937.078605:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695937.078607:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695937.078608:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695937.078611:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695937.078612:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695937.078613:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695937.078614:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695937.078615:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695937.078615:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695937.078616:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695937.078617:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695937.078619:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695937.078620:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695937.078686:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695937.078692:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695937.078696:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695937.462457:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695937.462463:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695937.462465:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695937.462469:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695937.462472:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695937.462475:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695937.462494:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695937.462496:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695937.462498:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695938.102451:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695938.102481:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695938.102487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695938.102488:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695938.102513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695938.102514:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695938.102517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695938.102518:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695938.102522:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695938.102522:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695938.102525:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695938.102526:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695938.102583:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695938.102589:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695938.102599:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787695938.422735:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695938.422746:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787695938.422749:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695938.422753:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695938.422759:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695938.422767:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3826 00000400:00000200:3.0:1787695938.422776:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695938.422783:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695938.422786:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695938.422790:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695938.422793:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695938.422796:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695938.422799:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695938.422803:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695938.422805:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:3.0:1787695938.422808:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695938.422811:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787695938.422814:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787695938.422816:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787695938.422820:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695938.422822:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695938.422824:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695938.422826:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695938.422829:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787695938.422833:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695938.422835:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695938.422838:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695938.422842:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695938.422852:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695938.422855:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787695938.422858:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695938.422864:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4a80. 00000020:00000040:3.0:1787695938.422868:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787695938.422872:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695938.422874:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695938.422877:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695938.422880:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695938.422884:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695938.422891:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675800. 00000100:00000001:3.0:1787695938.422898:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695938.422908:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4a80 x1874531139548288/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695938.422920:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695938.422924:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695938.422930:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ad0d67] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4a80 x1874531139548288/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695938.422954:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695938.422957:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695938.422961:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695938.422962:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695938.422964:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695938.422968:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695938.422970:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3826 00000400:00000200:3.0:1787695938.422975:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695938.422978:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695938.422981:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695938.422984:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695938.422987:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695938.422990:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695938.422998:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695938.423002:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787695938.423002:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695938.423004:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:3.0:1787695938.423007:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695938.423008:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1787695938.423010:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000100:00000001:1.0:1787695938.423013:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1787695938.423014:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:3.0:1787695938.423018:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:1.0:1787695938.423022:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4a80 x1874531139548288/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695938.423024:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695938.423028:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695938.423032:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695938.423037:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695938.423041:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00000001:1.0:1787695938.423043:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1787695938.423046:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695938.423048:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695938.423050:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695938.423055:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695938.423056:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695938.423060:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695938.423063:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787695938.423064:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787695938.423068:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1787695938.423070:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787695938.423075:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787695938.423076:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695938.423081:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695938.423087:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139548288:192.168.201.140@tcp:250:kworker.0 00000100:00000010:3.0:1787695938.423087:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5880. 00000020:00000040:3.0:1787695938.423092:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695938.423098:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787695938.423099:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787695938.423100:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695938.423103:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695938.423105:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787695938.423105:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695938.423109:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695938.423116:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675400. 02000000:00000001:1.0:1787695938.423117:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695938.423120:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695938.423128:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1787695938.423128:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5880 x1874531139548416/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:1.0:1787695938.423138:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545800. 02000000:00000001:1.0:1787695938.423142:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695938.423150:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695938.423157:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702108. 00000400:00000010:1.0:1787695938.423166:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46d48. 00000100:00000001:3.0:1787695938.423171:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000200:1.0:1787695938.423176:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139548288, portal 25 00000100:00000001:3.0:1787695938.423176:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695938.423182:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:3.0:1787695938.423185:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000827e7dbe] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5880 x1874531139548416/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695938.423187:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930146432 : -128555779405184 : ffff8b14466f4a80) 00000100:00000040:1.0:1787695938.423200:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4a80 x1874531139548288/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695938.423201:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695938.423205:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695938.423213:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695938.423217:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695938.423220:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695938.423224:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787695938.423225:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695938.423231:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3826 00000100:00000040:1.0:1787695938.423236:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000200:3.0:1787695938.423240:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695938.423242:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f467f8. 00000400:00000200:3.0:1787695938.423246:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000200:1.0:1787695938.423250:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139548288, offset 0 00000100:00080000:3.0:1787695938.423250:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695938.423257:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695938.423260:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695938.423264:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695938.423265:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695938.423270:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695938.423273:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695938.423274:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:3.0:1787695938.423276:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695938.423278:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:3.0:1787695938.423280:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:3.0:1787695938.423284:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000400:00000200:1.0:1787695938.423287:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695938.423290:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695938.423294:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695938.423297:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695938.423303:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695938.423308:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000040:3.0:1787695938.423308:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695938.423314:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695938.423316:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695938.423321:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000020:00000040:3.0:1787695938.423328:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:3.0:1787695938.423333:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:1.0:1787695938.423336:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17093:17093:256:4294967295] 192.168.201.140@tcp LPNI seq info [17092:17092:8:4294967295] 00000100:00000001:3.0:1787695938.423336:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695938.423339:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695938.423343:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695938.423347:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695938.423352:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695938.423352:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695938.423356:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695938.423360:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695938.423389:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:3.0:1787695938.423394:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695938.423399:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000010:3.0:1787695938.423401:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6680. 00000800:00000010:1.0:1787695938.423405:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981500. 00000020:00000040:3.0:1787695938.423405:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695938.423412:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695938.423415:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695938.423416:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787695938.423420:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695938.423424:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:1.1:1787695938.423425:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:3.0:1787695938.423429:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695938.423435:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 00000100:00000001:3.0:1787695938.423439:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695938.423442:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695938.423447:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6680 x1874531139548544/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695938.423449:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695938.423454:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298510137 00000100:00000001:3.0:1787695938.423463:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695938.423464:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298505017 00000100:00000001:3.0:1787695938.423467:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695938.423470:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298510137 00000100:00000040:3.0:1787695938.423475:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f6d2abf9] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6680 x1874531139548544/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695938.423476:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298505017 00000100:00000001:1.0:1787695938.423482:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695938.423503:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695938.423509:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695938.423510:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695938.423514:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695938.423516:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695938.423523:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787695938.423525:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695938.423527:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695938.423528:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695938.423531:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695938.423533:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787695938.423536:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787695938.423541:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3826 00000100:00000001:1.0:1787695938.423548:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000200:3.0:1787695938.423549:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695938.423552:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000400:00000200:3.0:1787695938.423555:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695938.423560:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695938.423561:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5880 x1874531139548416/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695938.423567:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695938.423572:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695938.423577:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695938.423581:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695938.423582:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695938.423585:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695938.423586:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695938.423588:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:3.0:1787695938.423590:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695938.423593:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:3.0:1787695938.423595:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:3.0:1787695938.423597:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:3.0:1787695938.423601:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695938.423606:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787695938.423607:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1787695938.423609:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695938.423611:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695938.423612:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:1.0:1787695938.423616:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695938.423618:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00100000:1.0:1787695938.423623:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139548416:192.168.201.140@tcp:38:kworker.0 00000100:00080000:3.0:1787695938.423624:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695938.423632:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695938.423633:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695938.423636:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787695938.423637:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695938.423639:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695938.423642:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787695938.423642:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:1.0:1787695938.423645:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546800. 02000000:00000001:3.0:1787695938.423647:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695938.423649:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695938.423651:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:1.0:1787695938.423655:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000008:00100000:3.0:1787695938.423655:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000400:00000010:1.0:1787695938.423660:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547022c0. 00000008:00000001:3.0:1787695938.423660:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695938.423665:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46e58. 00000100:00000001:3.0:1787695938.423665:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695938.423669:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5c00. 00000100:00000200:1.0:1787695938.423670:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139548416, portal 10 00000100:00000001:1.0:1787695938.423675:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000040:3.0:1787695938.423676:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000100:00000001:1.0:1787695938.423677:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150016 : -128555779401600 : ffff8b14466f5880) 02000000:00000001:3.0:1787695938.423682:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000040:1.0:1787695938.423686:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5880 x1874531139548416/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695938.423689:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695938.423694:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787695938.423698:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787695938.423698:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1787695938.423702:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695938.423702:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695938.423708:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46bb0. 02000000:00000010:3.0:1787695938.423710:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676000. 00000100:00000200:1.0:1787695938.423711:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139548416, offset 0 00000100:00000001:3.0:1787695938.423717:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695938.423719:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695938.423723:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695938.423726:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695938.423728:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695938.423730:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695938.423738:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695938.423739:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695938.423741:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17094:17094:255:4294967295] 192.168.201.140@tcp LPNI seq info [17093:17093:7:4294967295] 00000400:00000200:1.0:1787695938.423744:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695938.423750:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695938.423756:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695938.423760:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b1461981300. 00000800:00000200:1.0:1787695938.423764:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695938.423768:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:1.0:1787695938.423771:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695938.423775:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000400:00000200:1.0:1787695938.423781:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695938.423784:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298500038 00000100:00100000:3.0:1787695938.423787:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5c00 x1874531139548672/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695938.423789:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298510137 00000100:00000001:1.0:1787695938.423794:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298500038 00000100:00000001:1.0:1787695938.423800:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298510137 00000100:00000001:1.0:1787695938.423805:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695938.423806:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695938.423808:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695938.423810:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695938.423813:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1787695938.423818:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005bfbcee1] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5c00 x1874531139548672/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695938.423823:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6680 x1874531139548544/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695938.423837:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695938.423843:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1787695938.423844:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695938.423848:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695938.423850:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 02000000:00000001:1.0:1787695938.423854:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787695938.423854:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1787695938.423860:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1787695938.423860:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695938.423865:0:38:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1787695938.423866:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695938.423872:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3826 00000100:00100000:1.0:1787695938.423875:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139548544:192.168.201.140@tcp:38:kworker.0 00000400:00000200:3.0:1787695938.423881:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695938.423883:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695938.423888:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:3.0:1787695938.423889:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 02000000:00000001:1.0:1787695938.423891:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695938.423893:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695938.423897:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695938.423903:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787695938.423903:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695938.423908:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000010:1.0:1787695938.423909:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545400. 00000100:00000001:3.0:1787695938.423911:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:1.0:1787695938.423915:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695938.423916:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695938.423919:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:3.0:1787695938.423922:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:1.0:1787695938.423926:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000040:3.0:1787695938.423926:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:3.0:1787695938.423931:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000400:00000010:1.0:1787695938.423932:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702c08. 00000400:00000010:1.0:1787695938.423938:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46220. 00000100:00000200:1.0:1787695938.423941:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139548544, portal 10 00000020:00000001:3.0:1787695938.423945:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000001:1.0:1787695938.423946:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695938.423950:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930153600 : -128555779398016 : ffff8b14466f6680) 00000100:00000040:3.0:1787695938.423951:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695938.423957:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695938.423960:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695938.423964:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6680 x1874531139548544/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695938.423964:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695938.423970:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:3.0:1787695938.423976:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695938.423983:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787695938.423983:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695938.423987:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1787695938.423991:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695938.423992:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1787695938.423996:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f464c8. 02000000:00000001:3.0:1787695938.423997:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695938.424003:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139548544, offset 0 00000100:00000001:3.0:1787695938.424004:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695938.424007:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787695938.424013:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695938.424014:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695938.424017:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695938.424022:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000010:3.0:1787695938.424023:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6a00. 00000020:00000040:3.0:1787695938.424028:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 00000400:00000200:1.0:1787695938.424031:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 02000000:00000001:3.0:1787695938.424033:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695938.424036:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695938.424038:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1787695938.424039:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695938.424042:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1787695938.424044:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 02000000:00000001:3.0:1787695938.424045:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695938.424050:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 02000000:00000010:3.0:1787695938.424052:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 00000400:00000200:1.0:1787695938.424056:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:3.0:1787695938.424058:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695938.424064:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17095:17095:254:4294967295] 192.168.201.140@tcp LPNI seq info [17094:17094:6:4294967295] 00000100:00100000:3.0:1787695938.424069:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6a00 x1874531139548800/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695938.424075:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695938.424083:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695938.424088:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695938.424093:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000040:3.0:1787695938.424097:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002ad47e8f] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6a00 x1874531139548800/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1787695938.424104:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695938.424114:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da500. 00000100:00000001:3.0:1787695938.424114:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695938.424123:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00080000:3.0:1787695938.424125:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695938.424132:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000200:1.1:1787695938.424133:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787695938.424135:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787695938.424138:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695938.424139:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1787695938.424145:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000100:00000001:3.0:1787695938.424146:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695938.424152:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3826 00000400:00000200:1.0:1787695938.424154:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695938.424159:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695938.424164:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695938.424167:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298498886 00000100:00080000:3.0:1787695938.424168:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695938.424173:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298510137 00000100:00000040:3.0:1787695938.424174:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695938.424180:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298498886 00000100:00000001:3.0:1787695938.424181:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695938.424186:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298510137 00000100:00000040:3.0:1787695938.424186:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695938.424190:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695938.424193:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1787695938.424194:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695938.424198:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000100:00000001:1.0:1787695938.424200:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695938.424201:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695938.424205:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000100:00000001:1.0:1787695938.424207:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695938.424209:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:1.0:1787695938.424213:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787695938.424214:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:1.0:1787695938.424221:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:3.0:1787695938.424221:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695938.424225:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695938.424226:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000040:3.0:1787695938.424229:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695938.424232:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695938.424233:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695938.424236:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1787695938.424238:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00000001:1.0:1787695938.424240:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1787695938.424243:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695938.424246:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695938.424249:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1787695938.424250:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5c00 x1874531139548672/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695938.424253:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695938.424258:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695938.424260:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695938.424268:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695938.424274:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695938.424278:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695938.424285:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000010:3.0:1787695938.424285:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7b80. 00000100:00000001:1.0:1787695938.424288:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695938.424290:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:1.0:1787695938.424293:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787695938.424295:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695938.424298:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695938.424299:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695938.424303:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695938.424304:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695938.424311:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139548672:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695938.424315:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695938.424317:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695938.424319:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695938.424321:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695938.424326:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547400. 02000000:00000001:1.0:1787695938.424328:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695938.424332:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695938.424335:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547027e8. 00000400:00000010:1.0:1787695938.424340:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695938.424343:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139548672, portal 4 00000100:00000001:1.0:1787695938.424345:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695938.424352:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150912 : -128555779400704 : ffff8b14466f5c00) 00000100:00000040:1.0:1787695938.424360:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5c00 x1874531139548672/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695938.424391:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787695938.424392:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1787695938.424397:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695938.424398:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695938.424402:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46000. 02000000:00000010:3.0:1787695938.424405:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 00000100:00000200:1.0:1787695938.424408:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139548672, offset 0 00000100:00000001:3.0:1787695938.424411:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695938.424420:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00100000:3.0:1787695938.424422:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7b80 x1874531139548928/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695938.424430:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695938.424439:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:3.0:1787695938.424440:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695938.424444:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695938.424446:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1787695938.424452:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b87e1da] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7b80 x1874531139548928/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695938.424454:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695938.424462:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695938.424469:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:3.0:1787695938.424471:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695938.424476:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17096:17096:253:4294967295] 192.168.201.140@tcp LPNI seq info [17095:17095:5:4294967295] 00000100:00080000:3.0:1787695938.424477:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695938.424499:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:3.0:1787695938.424500:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695938.424504:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695938.424507:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695938.424514:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1787695938.424517:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00080000:3.0:1787695938.424520:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3826 00000800:00000200:1.0:1787695938.424530:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000400:00000200:3.0:1787695938.424530:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695938.424535:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000800:00000010:1.0:1787695938.424536:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460daa00. 00000100:00080000:3.0:1787695938.424540:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1787695938.424547:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:3.0:1787695938.424547:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695938.424551:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695938.424555:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000200:1.1:1787695938.424558:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:1.0:1787695938.424563:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695938.424565:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000040:1.0:1787695938.424569:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000020:00000001:3.0:1787695938.424570:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695938.424572:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000020:00000001:3.0:1787695938.424575:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:1.0:1787695938.424579:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000040:3.0:1787695938.424579:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000100:00000001:1.0:1787695938.424584:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298498917 00000020:00000001:3.0:1787695938.424584:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000020:00000001:3.0:1787695938.424589:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:1.0:1787695938.424591:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298510137 00000100:00000040:3.0:1787695938.424596:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695938.424598:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298498917 00000100:00000001:3.0:1787695938.424600:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695938.424604:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298510137 00000100:00000040:3.0:1787695938.424605:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695938.424609:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695938.424611:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1787695938.424613:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00000001:1.0:1787695938.424617:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695938.424619:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695938.424623:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787695938.424623:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695938.424626:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695938.424629:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695938.424632:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695938.424635:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695938.424639:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000040:1.0:1787695938.424640:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6a00 x1874531139548800/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1787695938.424643:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695938.424647:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695938.424652:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7800. 00000020:00000040:3.0:1787695938.424657:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000001:1.0:1787695938.424659:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787695938.424662:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787695938.424663:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695938.424665:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695938.424668:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787695938.424669:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695938.424673:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695938.424674:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695938.424675:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695938.424679:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695938.424682:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 00000100:00100000:1.0:1787695938.424686:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139548800:192.168.201.140@tcp:8:kworker.0 00000100:00000001:3.0:1787695938.424688:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695938.424695:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1787695938.424699:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7800 x1874531139549056/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787695938.424700:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695938.424703:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695938.424709:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695938.424715:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545c00. 02000000:00000001:1.0:1787695938.424720:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695938.424730:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695938.424734:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1787695938.424736:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000100:00000001:3.0:1787695938.424738:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695938.424745:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000040:3.0:1787695938.424746:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7c57981] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7800 x1874531139549056/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695938.424752:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139548800, portal 4 00000100:00000001:1.0:1787695938.424758:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695938.424761:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930154496 : -128555779397120 : ffff8b14466f6a00) 00000100:00000001:3.0:1787695938.424762:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695938.424768:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1787695938.424774:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6a00 x1874531139548800/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695938.424774:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695938.424779:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787695938.424786:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1787695938.424793:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1787695938.424794:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3836) 00000100:00000040:1.0:1787695938.424799:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695938.424803:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46f68. 00000100:00000200:1.0:1787695938.424806:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139548800, offset 0 00000400:00000200:1.0:1787695938.424811:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695938.424815:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695938.424819:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695938.424822:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695938.424825:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695938.424827:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695938.424830:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695938.424841:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17097:17097:252:4294967295] 192.168.201.140@tcp LPNI seq info [17096:17096:4:4294967295] 00000400:00000200:1.0:1787695938.424845:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695938.424852:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695938.424856:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695938.424861:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da700. 00000800:00000200:1.0:1787695938.424864:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695938.424869:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:1.0:1787695938.424876:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695938.424879:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695938.424882:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695938.424884:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298498589 00000100:00000001:1.0:1787695938.424887:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298510137 00000100:00000001:1.0:1787695938.424889:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298498589 00000100:00000001:1.0:1787695938.424891:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298510137 00000100:00000001:1.0:1787695938.424894:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695938.424896:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695938.424899:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695938.424902:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695938.424906:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695938.424908:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787695938.424910:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695938.424911:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695938.424913:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695938.424917:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7b80 x1874531139548928/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695938.424931:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695938.424933:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695938.424935:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695938.424937:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695938.424940:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695938.424943:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139548928:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695938.424954:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695938.424957:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695938.424958:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695938.424961:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695938.424964:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545000. 02000000:00000001:1.0:1787695938.424966:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695938.424970:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695938.424978:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702688. 00000400:00000010:1.0:1787695938.424982:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46330. 00000100:00000200:1.0:1787695938.424986:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139548928, portal 10 00000100:00000001:1.0:1787695938.424988:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695938.424990:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930158976 : -128555779392640 : ffff8b14466f7b80) 00000100:00000040:1.0:1787695938.424996:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7b80 x1874531139548928/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695938.425004:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695938.425006:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695938.425009:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46cc0. 00000100:00000200:1.0:1787695938.425011:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139548928, offset 0 00000400:00000200:1.0:1787695938.425014:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695938.425017:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695938.425019:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695938.425021:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695938.425023:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695938.425026:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695938.425029:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695938.425032:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17098:17098:251:4294967295] 192.168.201.140@tcp LPNI seq info [17097:17097:3:4294967295] 00000400:00000200:1.0:1787695938.425036:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695938.425042:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695938.425047:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695938.425051:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da800. 00000800:00000200:1.0:1787695938.425055:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695938.425065:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:1.0:1787695938.425068:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695938.425071:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695938.425075:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695938.425077:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298498775 00000100:00000001:1.0:1787695938.425080:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298510137 00000100:00000001:1.0:1787695938.425082:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298498775 00000100:00000001:1.0:1787695938.425084:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298510137 00000100:00000001:1.0:1787695938.425085:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695938.425087:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695938.425088:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695938.425092:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7800 x1874531139549056/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695938.425097:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695938.425098:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695938.425099:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695938.425100:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695938.425102:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695938.425104:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139549056:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695938.425106:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695938.425107:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695938.425109:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695938.425110:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695938.425112:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544c00. 02000000:00000001:1.0:1787695938.425113:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695938.425115:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695938.425117:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b1454702630. 00000400:00000010:1.0:1787695938.425120:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f463b8. 00000100:00000200:1.0:1787695938.425122:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139549056, portal 10 00000100:00000001:1.0:1787695938.425123:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695938.425124:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930158080 : -128555779393536 : ffff8b14466f7800) 00000100:00000040:1.0:1787695938.425130:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7800 x1874531139549056/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695938.425142:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695938.425145:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695938.425148:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f465d8. 00000100:00000200:1.0:1787695938.425151:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139549056, offset 0 00000400:00000200:1.0:1787695938.425155:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695938.425160:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695938.425163:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695938.425166:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695938.425169:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695938.425172:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695938.425174:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695938.425178:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17099:17099:250:4294967295] 192.168.201.140@tcp LPNI seq info [17098:17098:2:4294967295] 00000400:00000200:1.0:1787695938.425182:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695938.425189:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695938.425192:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695938.425195:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695938.425198:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695938.425200:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:1.0:1787695938.425202:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695938.425204:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695938.425207:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695938.425208:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298498753 00000100:00000001:1.0:1787695938.425210:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298510137 00000100:00000001:1.0:1787695938.425211:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298498753 00000100:00000001:1.0:1787695938.425213:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298510137 00000100:00000001:1.0:1787695938.425214:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695938.425216:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695938.425219:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695938.425221:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695938.425224:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695938.425231:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787695938.425233:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695938.425235:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695938.425237:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695938.425239:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695938.486457:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695938.486463:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695938.486465:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695938.486469:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695938.486472:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695938.486476:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695938.486478:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695938.486479:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695938.486503:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695939.126524:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695939.126530:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695939.126536:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695939.126537:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695939.126541:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695939.126542:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695939.126546:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695939.126548:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695939.126551:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695939.126552:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695939.126556:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695939.126557:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695939.126617:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695939.126620:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695939.126623:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695939.510440:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695939.510446:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695939.510448:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695939.510452:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695939.510455:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695939.510458:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695939.510460:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695939.510461:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695939.510464:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695940.150444:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695940.150450:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695940.150477:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695940.150478:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695940.150481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695940.150482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695940.150485:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695940.150486:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695940.150488:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695940.150489:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695940.150491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695940.150492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695940.150550:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695940.150556:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695940.150560:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695940.534425:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695940.534432:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695940.534434:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695940.534438:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695940.534441:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695940.534444:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695940.534446:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695940.534473:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695940.534475:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695941.174522:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695941.174527:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695941.174532:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695941.174533:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695941.174536:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695941.174536:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695941.174539:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695941.174540:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695941.174543:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695941.174559:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695941.174561:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695941.174566:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695941.174568:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695941.174569:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695941.174571:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695941.174573:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695941.174575:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695941.174576:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695941.174578:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695941.174579:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695941.174641:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695941.174645:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695941.174649:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787695941.494868:0:2390:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.140:988 00000400:00000100:0.0:1787695941.494926:0:2390:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.140@tcp at host 192.168.201.140 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1787695941.494946:0:2390:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695941.494951:0:2390:0:(router.c:1791:lnet_notify()) 192.168.201.40@tcp notifying 192.168.201.140@tcp: down 00000400:00000200:0.0:1787695941.494956:0:2390:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787695941.494980:0:2390:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1787695941.494988:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:0.0:1787695941.494993:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981500. 00000400:00000200:0.0:1787695941.495001:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1787695941.495007:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1787695941.495015:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f467f8 00000400:00000010:0.0:1787695941.495018:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f467f8. 00000100:00000001:0.0:1787695941.495025:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695941.495037:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f4a80 x1874531139548288/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695941.495067:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695941.495072:0:2390:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4a80 x1874531139548288/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695963 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695941.495081:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695941.495083:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1787695941.495096:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:0.0:1787695941.495099:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b1461981300. 00000400:00000200:0.0:1787695941.495102:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1787695941.495106:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1787695941.495110:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46bb0 00000400:00000010:0.0:1787695941.495112:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46bb0. 00000100:00000001:0.0:1787695941.495116:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695941.495123:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f5880 x1874531139548416/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695941.495133:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787695941.495138:0:2390:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5880 x1874531139548416/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695963 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695941.495145:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695941.495147:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1787695941.495156:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000100:00000001:1.0:1787695941.495156:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000010:0.0:1787695941.495160:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da500. 00000400:00000200:0.0:1787695941.495168:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:1.0:1787695941.495168:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695941.495177:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000400:00000200:0.0:1787695941.495181:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1787695941.495185:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f464c8 00000400:00000010:0.0:1787695941.495187:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f464c8. 00000100:00000001:0.0:1787695941.495190:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695941.495197:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f6680 x1874531139548544/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695941.495209:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:1.0:1787695941.495211:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695938/real 1787695941] req@ffff8b14466f5880 x1874531139548416/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695941.495220:0:2390:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6680 x1874531139548544/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695963 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695941.495290:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695941.495293:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1787695941.495299:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000400:00000001:1.0:1787695941.495303:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000010:0.0:1787695941.495304:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460daa00. 00000400:00000001:1.0:1787695941.495311:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695941.495317:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547022c0. 00000400:00000200:1.0:1787695941.495325:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46e58 00000400:00000200:0.0:1787695941.495327:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000010:1.0:1787695941.495330:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46e58. 00000400:00000200:0.0:1787695941.495340:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000100:00000001:1.0:1787695941.495342:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:0.0:1787695941.495346:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46000 00000400:00000010:0.0:1787695941.495355:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46000. 00000100:00000200:1.0:1787695941.495356:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5880 x1874531139548416/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695941.495429:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5880 x1874531139548416/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695941.495448:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787695941.495454:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000200:0.0:1787695941.495461:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f5c00 x1874531139548672/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.495462:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.495471:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695941.495474:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695941.495491:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:1.0:1787695941.495494:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5880 x1874531139548416/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695941.495501:0:2390:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5c00 x1874531139548672/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695963 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.495536:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695941.495539:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787695941.495542:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1787695941.495550:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:0.0:1787695941.495553:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da700. 00000400:00000200:0.0:1787695941.495557:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1787695941.495562:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1787695941.495573:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46f68 00000400:00000010:0.0:1787695941.495576:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46f68. 00000100:00000040:1.0:1787695941.495580:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5880 x1874531139548416/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695941.495595:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787695941.495600:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000200:0.0:1787695941.495607:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f6a00 x1874531139548800/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.495611:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695941.495625:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787695941.495631:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:0.0:1787695941.495633:0:2390:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6a00 x1874531139548800/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695963 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.495638:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.495645:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695941.495650:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787695941.495653:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695941.495658:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1787695941.495659:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:1.0:1787695941.495673:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000800:00000100:0.0:1787695941.495675:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000100:00000001:1.0:1787695941.495683:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000800:00000010:0.0:1787695941.495684:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da800. 00000400:00000200:0.0:1787695941.495696:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000040:1.0:1787695941.495702:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5880 x1874531139548416/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695941.495710:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1787695941.495720:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46cc0 00000400:00000010:0.0:1787695941.495724:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46cc0. 00000100:00000001:0.0:1787695941.495732:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:1.0:1787695941.495739:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139548416:192.168.201.140@tcp:38:kworker.0 00000100:00000200:0.0:1787695941.495742:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f7b80 x1874531139548928/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.495751:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787695941.495760:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:1.0:1787695941.495763:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695938/real 1787695941] req@ffff8b14466f6680 x1874531139548544/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1787695941.495770:0:2390:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7b80 x1874531139548928/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695963 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695941.495831:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787695941.495831:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695941.495836:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787695941.495837:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000010:1.0:1787695941.495841:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702c08. 00000400:00000200:1.0:1787695941.495849:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46220 00000800:00000100:0.0:1787695941.495850:0:2390:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000400:00000010:1.0:1787695941.495857:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46220. 00000800:00000010:0.0:1787695941.495858:0:2390:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:0.0:1787695941.495870:0:2390:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000100:00000001:1.0:1787695941.495872:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:0.0:1787695941.495886:0:2390:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000100:00000200:1.0:1787695941.495886:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6680 x1874531139548544/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1787695941.495895:0:2390:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f465d8 00000400:00000010:0.0:1787695941.495903:0:2390:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f465d8. 00000100:00000001:0.0:1787695941.495911:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787695941.495919:0:2390:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b14466f7800 x1874531139549056/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695963 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695941.495928:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6680 x1874531139548544/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.495947:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787695941.495961:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787695941.495963:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.495969:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:0.0:1787695941.495971:0:2390:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7800 x1874531139549056/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695963 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.495973:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787695941.495989:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695941.495992:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6680 x1874531139548544/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787695941.495994:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695941.496006:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695941.496014:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6680 x1874531139548544/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496021:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695941.496024:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496025:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695941.496027:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496029:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695941.496031:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695941.496033:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695941.496037:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695941.496040:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695941.496044:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6680 x1874531139548544/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695941.496052:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139548544:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695941.496057:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695941.496063:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695938/real 1787695941] req@ffff8b14466f4a80 x1874531139548288/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695941.496070:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695941.496072:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695941.496074:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702108. 00000400:00000200:1.0:1787695941.496078:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46d48 00000400:00000010:1.0:1787695941.496080:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46d48. 00000100:00000001:1.0:1787695941.496090:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695941.496094:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4a80 x1874531139548288/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695941.496102:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4a80 x1874531139548288/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496109:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695941.496110:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496112:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695941.496114:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695941.496118:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4a80 x1874531139548288/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496124:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695941.496128:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4a80 x1874531139548288/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496134:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695941.496137:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496139:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695941.496140:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496142:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695941.496143:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695941.496146:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695941.496150:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695941.496152:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695941.496156:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4a80 x1874531139548288/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695941.496163:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139548288:192.168.201.140@tcp:250:kworker.0 00000100:00000001:1.0:1787695941.496168:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695941.496171:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695941.496180:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5880 x1874531139548416/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496187:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695941.496191:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695941.496196:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546800. 02000000:00000001:1.0:1787695941.496205:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695941.496208:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695941.496211:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695941.496214:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695941.496217:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675400. 02000000:00000001:1.0:1787695941.496222:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695941.496225:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695941.496228:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5880. 00000100:00000001:1.0:1787695941.496231:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695941.496233:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496236:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695941.496240:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6680 x1874531139548544/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496247:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695941.496249:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695941.496251:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787695941.496256:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695941.496257:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695941.496258:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695941.496261:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695941.496263:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:1.0:1787695941.496270:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695941.496272:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695941.496274:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6680. 00000100:00000001:1.0:1787695941.496278:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695941.496279:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496281:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695941.496285:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4a80 x1874531139548288/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496291:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695941.496299:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695941.496301:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545800. 02000000:00000001:1.0:1787695941.496304:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695941.496306:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695941.496307:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:1.0:1787695941.496309:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695941.496310:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675800. 02000000:00000001:1.0:1787695941.496313:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695941.496314:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695941.496315:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4a80. 00000100:00000001:1.0:1787695941.496317:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695941.496319:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496322:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695941.496324:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695941.496326:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695941.496328:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695941.496333:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695938/real 1787695941] req@ffff8b14466f7b80 x1874531139548928/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695941.496342:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695941.496343:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695941.496345:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702688. 00000400:00000200:1.0:1787695941.496348:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46330 00000400:00000010:1.0:1787695941.496350:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46330. 00000100:00000001:1.0:1787695941.496352:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695941.496355:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7b80 x1874531139548928/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695941.496395:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7b80 x1874531139548928/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496404:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695941.496406:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496409:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695941.496410:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695941.496415:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7b80 x1874531139548928/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496449:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695941.496455:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7b80 x1874531139548928/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496462:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695941.496465:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496467:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695941.496469:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496471:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695941.496475:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695941.496479:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695941.496483:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695941.496486:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695941.496491:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7b80 x1874531139548928/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695941.496500:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139548928:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695941.496507:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695941.496513:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695938/real 1787695941] req@ffff8b14466f7800 x1874531139549056/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695941.496524:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695941.496526:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695941.496529:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b1454702630. 00000400:00000200:1.0:1787695941.496533:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f463b8 00000400:00000010:1.0:1787695941.496535:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f463b8. 00000100:00000001:1.0:1787695941.496537:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695941.496542:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7800 x1874531139549056/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695941.496568:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7800 x1874531139549056/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496574:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695941.496576:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496578:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695941.496579:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695941.496583:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7800 x1874531139549056/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496589:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695941.496593:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7800 x1874531139549056/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496599:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695941.496602:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496604:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695941.496605:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496607:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695941.496609:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695941.496612:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695941.496615:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695941.496617:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695941.496621:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7800 x1874531139549056/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695941.496629:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139549056:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695941.496633:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695941.496638:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695938/real 1787695941] req@ffff8b14466f5c00 x1874531139548672/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695941.496659:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695941.496661:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695941.496663:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547027e8. 00000400:00000200:1.0:1787695941.496666:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695941.496668:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695941.496672:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695941.496679:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5c00 x1874531139548672/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695941.496687:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5c00 x1874531139548672/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496693:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695941.496694:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496696:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695941.496697:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695941.496699:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5c00 x1874531139548672/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496704:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695941.496706:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5c00 x1874531139548672/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496711:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695941.496713:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496713:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695941.496714:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496715:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695941.496717:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695941.496719:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695941.496721:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695941.496722:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695941.496731:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5c00 x1874531139548672/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695941.496740:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139548672:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695941.496743:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695941.496748:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695938/real 1787695941] req@ffff8b14466f6a00 x1874531139548800/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695941.496756:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695941.496758:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695941.496760:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695941.496762:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695941.496764:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695941.496767:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695941.496770:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6a00 x1874531139548800/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695941.496779:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6a00 x1874531139548800/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496787:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695941.496789:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496791:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695941.496792:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695941.496796:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6a00 x1874531139548800/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496802:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695941.496805:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6a00 x1874531139548800/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496809:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695941.496810:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496818:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695941.496820:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496822:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695941.496824:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695941.496826:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695941.496830:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695941.496833:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695941.496837:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6a00 x1874531139548800/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695941.496845:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139548800:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695941.496850:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496852:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695941.496855:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7b80 x1874531139548928/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496863:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695941.496865:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695941.496867:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545000. 02000000:00000001:1.0:1787695941.496870:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695941.496872:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695941.496873:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695941.496876:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695941.496878:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:1.0:1787695941.496882:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695941.496883:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695941.496885:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7b80. 00000100:00000001:1.0:1787695941.496888:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695941.496890:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496891:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695941.496895:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7800 x1874531139549056/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496901:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695941.496909:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695941.496911:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544c00. 02000000:00000001:1.0:1787695941.496915:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695941.496916:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695941.496918:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695941.496919:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695941.496921:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:1.0:1787695941.496924:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695941.496925:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695941.496926:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7800. 00000100:00000001:1.0:1787695941.496929:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695941.496930:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496932:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695941.496937:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5c00 x1874531139548672/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496944:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695941.496946:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695941.496947:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547400. 02000000:00000001:1.0:1787695941.496951:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695941.496953:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695941.496954:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695941.496956:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695941.496957:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676000. 02000000:00000001:1.0:1787695941.496960:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695941.496961:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695941.496963:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5c00. 00000100:00000001:1.0:1787695941.496966:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695941.496967:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496968:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695941.496971:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6a00 x1874531139548800/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695963 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695941.496975:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695941.496976:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695941.496977:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545c00. 02000000:00000001:1.0:1787695941.496979:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695941.496980:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695941.496985:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695941.496987:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695941.496988:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:1.0:1787695941.496990:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695941.496991:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695941.496992:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6a00. 00000100:00000001:1.0:1787695941.496994:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695941.496995:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.496996:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695941.497003:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695941.497004:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695941.497006:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695941.497007:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695941.558472:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695941.558481:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695941.558484:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695941.558489:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695941.558492:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695941.558496:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695941.558499:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695941.558501:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695941.558504:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695942.198494:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695942.198500:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695942.198504:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695942.198505:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695942.198508:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695942.198509:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695942.198511:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695942.198512:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695942.198514:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695942.198515:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695942.198518:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695942.198519:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695942.198586:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695942.198589:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695942.198592:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695942.582454:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695942.582461:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695942.582463:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695942.582468:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695942.582471:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695942.582474:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695942.582476:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695942.582478:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695942.582479:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695943.222447:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695943.222454:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695943.222456:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695943.222459:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695943.222463:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695943.222465:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695943.222466:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695943.222468:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695943.222471:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695943.222472:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695943.222473:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695943.222474:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695943.222475:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695943.222493:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695943.222494:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695943.222495:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695943.222498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695943.222499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695943.222502:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695943.222503:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695943.222505:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695943.222506:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695943.222508:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695943.222509:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695943.222566:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695943.222572:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695943.222576:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787695943.542781:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695943.542798:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787695943.542803:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695943.542807:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695943.542814:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695943.542823:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3831 00000400:00000200:1.0:1787695943.542835:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695943.542843:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695943.542925:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695943.542932:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.542936:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695943.542939:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.542942:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695943.542948:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695943.542950:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:1.0:1787695943.542954:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695943.542958:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:1.0:1787695943.542961:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:1.0:1787695943.542964:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:1.0:1787695943.542968:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.542970:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695943.542986:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.542988:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695943.542992:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:1.0:1787695943.542996:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695943.542999:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695943.543003:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695943.543008:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695943.543011:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695943.543015:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695943.543018:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695943.543026:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446666300. 00000020:00000040:1.0:1787695943.543031:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:1.0:1787695943.543037:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695943.543039:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695943.543044:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695943.543047:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695943.543052:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695943.543062:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545c00. 00000100:00000001:1.0:1787695943.543072:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695943.543087:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446666300 x1874531139549184/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695943.543100:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695943.543105:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695943.543112:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000945f4e20] to pc [ptlrpcd_rcv+-1] req@ffff8b1446666300 x1874531139549184/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695943.543246:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695943.543251:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695943.543255:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787695943.543256:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695943.543258:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695943.543261:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695943.543275:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3831 00000400:00000200:1.0:1787695943.543281:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695943.543285:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695943.543287:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695943.543290:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.543292:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695943.543294:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.543297:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695943.543299:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695943.543301:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:1.0:1787695943.543302:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695943.543305:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:1.0:1787695943.543307:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:1.0:1787695943.543309:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:1.0:1787695943.543312:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.543315:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695943.543317:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.543319:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695943.543322:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:1.0:1787695943.543325:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695943.543335:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695943.543338:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695943.543345:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695943.543347:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695943.543350:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787695943.543356:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787695943.543359:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695943.543361:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695943.543441:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446664e00. 00000020:00000040:1.0:1787695943.543444:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:1.0:1787695943.543447:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695943.543454:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695943.543457:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695943.543458:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695943.543461:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695943.543467:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446547400. 00000100:00000001:1.0:1787695943.543471:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695943.543478:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446664e00 x1874531139549312/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695943.543487:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695943.543490:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695943.543494:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b012f68f] to pc [ptlrpcd_rcv+-1] req@ffff8b1446664e00 x1874531139549312/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695943.543503:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695943.543507:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695943.543511:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787695943.543513:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695943.543514:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695943.543518:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695943.543520:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3831 00000400:00000200:1.0:1787695943.543525:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695943.543529:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695943.543531:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695943.543535:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.543537:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695943.543539:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.543541:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695943.543544:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695943.543545:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:1.0:1787695943.543547:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695943.543549:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:1.0:1787695943.543551:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:1.0:1787695943.543560:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:1.0:1787695943.543564:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.543566:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695943.543568:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.543570:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695943.543572:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:1.0:1787695943.543575:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695943.543578:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695943.543579:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695943.543583:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695943.543585:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695943.543587:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787695943.543590:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787695943.543592:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695943.543593:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695943.543597:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446664380. 00000020:00000040:1.0:1787695943.543599:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:1.0:1787695943.543602:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695943.543604:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695943.543605:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695943.543607:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695943.543609:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695943.543613:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446544c00. 00000100:00000001:1.0:1787695943.543616:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695943.543622:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446664380 x1874531139549440/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695943.543630:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695943.543632:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695943.543635:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000094a36b85] to pc [ptlrpcd_rcv+-1] req@ffff8b1446664380 x1874531139549440/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695943.543644:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695943.543653:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695943.543657:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787695943.543659:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695943.543661:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695943.543664:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695943.543668:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3831 00000400:00000200:1.0:1787695943.543673:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695943.543676:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695943.543679:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695943.543682:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.543684:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695943.543686:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.543689:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695943.543692:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695943.543693:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:1.0:1787695943.543695:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695943.543697:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:1.0:1787695943.543700:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:1.0:1787695943.543702:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:1.0:1787695943.543706:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.543708:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695943.543710:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.543712:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695943.543715:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:1.0:1787695943.543718:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695943.543720:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695943.543722:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695943.543726:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695943.543729:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695943.543731:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787695943.543741:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787695943.543744:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695943.543746:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695943.543751:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446666a00. 00000020:00000040:1.0:1787695943.543754:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:1.0:1787695943.543757:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695943.543758:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695943.543760:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695943.543762:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695943.543764:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695943.543768:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545000. 00000100:00000001:1.0:1787695943.543772:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695943.543778:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446666a00 x1874531139549568/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695943.543786:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695943.543789:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695943.543792:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000590093d2] to pc [ptlrpcd_rcv+-1] req@ffff8b1446666a00 x1874531139549568/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695943.543801:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695943.543804:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695943.543809:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1787695943.543811:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695943.543812:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695943.543816:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695943.543818:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3831 00000400:00000200:1.0:1787695943.543823:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695943.543826:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695943.543829:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695943.543832:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.543835:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695943.543837:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.543848:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695943.543850:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695943.543862:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:1.0:1787695943.543864:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695943.543867:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:1.0:1787695943.543870:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:1.0:1787695943.543872:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:1.0:1787695943.543876:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.543878:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695943.543880:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.543882:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695943.543884:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:1.0:1787695943.543887:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695943.543890:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695943.543892:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695943.543895:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695943.543896:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695943.543898:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787695943.543901:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787695943.543903:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695943.543905:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695943.543909:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665c00. 00000020:00000040:1.0:1787695943.543911:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:1.0:1787695943.543914:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695943.543916:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695943.543917:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695943.543919:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695943.543921:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695943.543925:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545800. 00000100:00000001:1.0:1787695943.543929:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695943.543933:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446665c00 x1874531139549696/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695943.543952:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695943.543954:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695943.543958:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d782cc4] to pc [ptlrpcd_rcv+-1] req@ffff8b1446665c00 x1874531139549696/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695943.543966:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695943.543969:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695943.543973:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787695943.543975:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695943.543976:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695943.543979:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695943.543982:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3831 00000400:00000200:1.0:1787695943.543986:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695943.543989:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695943.543991:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695943.543994:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.543997:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695943.543999:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.544001:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695943.544004:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695943.544005:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:1.0:1787695943.544006:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695943.544009:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000020:00000001:1.0:1787695943.544011:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000020:00000001:1.0:1787695943.544013:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000040:1.0:1787695943.544017:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.544019:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695943.544021:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.544023:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695943.544034:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:1.0:1787695943.544037:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695943.544040:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695943.544042:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695943.544045:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695943.544047:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695943.544050:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787695943.544052:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787695943.544055:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695943.544057:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695943.544060:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446666d80. 00000020:00000040:1.0:1787695943.544063:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:1.0:1787695943.544065:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695943.544067:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695943.544069:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695943.544070:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695943.544072:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695943.544076:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545400. 00000100:00000001:1.0:1787695943.544080:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695943.544085:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446666d80 x1874531139549824/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695943.544093:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695943.544095:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695943.544099:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa82a36] to pc [ptlrpcd_rcv+-1] req@ffff8b1446666d80 x1874531139549824/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695943.544108:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695943.544111:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695943.544114:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787695943.544116:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695943.544118:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695943.544121:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695943.544130:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3831 00000400:00000200:1.0:1787695943.544135:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695943.544138:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695943.544141:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695943.544144:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.544146:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695943.544148:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.544150:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695943.544152:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695943.544154:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000020:00000001:1.0:1787695943.544156:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695943.544158:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:1.0:1787695943.544160:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000020:00000001:1.0:1787695943.544162:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:1.0:1787695943.544165:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.544168:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695943.544170:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695943.544172:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695943.544175:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00080000:1.0:1787695943.544178:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695943.544182:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695943.544184:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695943.544188:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695943.544191:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695943.544193:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787695943.544196:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787695943.544198:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695943.544200:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695943.544201:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695943.544208:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446667480. 00000100:00000001:2.0:1787695943.544208:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000040:1.0:1787695943.544211:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000001:2.0:1787695943.544214:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1787695943.544215:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695943.544222:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695943.544224:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:2.0:1787695943.544226:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446666300 x1874531139549184/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695943.544232:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695943.544236:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695943.544242:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446546800. 00000100:00000001:2.0:1787695943.544244:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695943.544246:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695943.544249:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695943.544254:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446667480 x1874531139549952/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787695943.544258:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695943.544264:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695943.544265:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695943.544267:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695943.544270:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000390bc45a] to pc [ptlrpcd_rcv+-1] req@ffff8b1446667480 x1874531139549952/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695943.544280:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695943.544283:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695943.544287:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787695943.544290:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446666300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139549184:192.168.201.140@tcp:250:kworker.0 00000100:00000040:1.0:1787695943.544291:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:1.0:1787695943.544297:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787695943.544300:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1787695943.544301:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 02000000:00000001:2.0:1787695943.544307:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:1.0:1787695943.544308:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3841) 02000000:00000001:2.0:1787695943.544314:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695943.544319:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695943.544326:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445660000. 02000000:00000001:2.0:1787695943.544329:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695943.544337:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695943.544344:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695943.544353:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695943.544361:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139549184, portal 25 00000100:00000001:2.0:1787695943.544439:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695943.544441:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929562880 : -128555779988736 : ffff8b1446666300) 00000100:00000040:2.0:1787695943.544449:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446666300 x1874531139549184/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695968 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.544459:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695943.544462:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695943.544474:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695943.544482:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139549184, offset 0 00000400:00000200:2.0:1787695943.544490:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695943.544501:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695943.544509:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695943.544516:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695943.544521:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695943.544527:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695943.544531:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695943.544535:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17100:17100:256:4294967295] 192.168.201.140@tcp LPNI seq info [17099:17099:8:4294967295] 00000400:00000200:2.0:1787695943.544540:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695943.544552:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695943.544560:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695943.544566:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695943.544591:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695943.544596:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1787695943.544602:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695943.544606:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1787695943.544611:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695943.544614:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695943.544620:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695943.544625:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695943.544631:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695943.544634:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695943.544639:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695943.544643:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446666300 x1874531139549184/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695968 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.544654:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695943.544658:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446666300 x1874531139549184/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695968 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.544666:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695943.544668:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695943.544675:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695943.544679:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695943.544683:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298515258 00000100:00000001:2.0:1787695943.544691:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298510137 00000100:00000001:2.0:1787695943.544696:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298515258 00000100:00000001:2.0:1787695943.544699:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298510137 00000100:00000001:2.0:1787695943.544702:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695943.544706:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695943.544708:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695943.544713:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695943/real 1787695943] req@ffff8b1446666300 x1874531139549184/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695943.544723:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695943.544725:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695943.544727:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695943.544735:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695943.544737:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695943.544740:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695943.544744:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446666300 x1874531139549184/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695943.544752:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446666300 x1874531139549184/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.544759:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695943.544761:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.544764:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695943.544766:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695943.544770:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446666300 x1874531139549184/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.544776:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695943.544781:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446666300 x1874531139549184/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.544788:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695943.544792:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.544793:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695943.544794:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.544797:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695943.544799:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695943.544801:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695943.544805:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695943.544808:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695943.544812:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446666300 x1874531139549184/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695943.544820:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446666300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139549184:192.168.201.140@tcp:250:kworker.0 00000100:00000001:2.0:1787695943.544825:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695943.544835:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446664e00 x1874531139549312/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.544841:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695943.544843:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695943.544846:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695943.544847:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695943.544849:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695943.544852:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446664e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139549312:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695943.544856:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695943.544858:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695943.544860:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695943.544862:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695943.544865:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445662400. 02000000:00000001:2.0:1787695943.544867:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695943.544872:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695943.544875:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695943.544879:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695943.544882:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139549312, portal 10 00000100:00000001:2.0:1787695943.544884:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695943.544885:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929557504 : -128555779994112 : ffff8b1446664e00) 00000100:00000040:2.0:1787695943.544893:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446664e00 x1874531139549312/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695968 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.544899:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695943.544901:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695943.544903:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695943.544906:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139549312, offset 0 00000400:00000200:2.0:1787695943.544909:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695943.544913:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695943.544916:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695943.544919:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695943.544922:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695943.544924:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695943.544932:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695943.544935:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17101:17101:256:4294967295] 192.168.201.140@tcp LPNI seq info [17100:17100:8:4294967295] 00000400:00000200:2.0:1787695943.544939:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695943.544946:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695943.544950:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695943.544953:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695943.544957:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695943.544960:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1787695943.544964:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695943.544967:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1787695943.544971:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695943.544973:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695943.544976:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695943.544979:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695943.544982:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695943.544984:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695943.544987:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695943.544991:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446664e00 x1874531139549312/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695968 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.544999:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695943.545002:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446664e00 x1874531139549312/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695968 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545009:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695943.545010:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695943.545013:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695943.545017:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000400:00000200:2.0:1787695943.545020:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695943.545023:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298505159 00000100:00000001:2.0:1787695943.545026:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298515258 00000100:00000001:2.0:1787695943.545028:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298505159 00000100:00000001:2.0:1787695943.545031:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298515258 00000100:00000001:2.0:1787695943.545041:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695943.545043:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695943.545044:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695943.545049:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695943/real 1787695943] req@ffff8b1446664e00 x1874531139549312/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695943.545058:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695943.545059:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695943.545061:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695943.545063:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695943.545065:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695943.545067:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695943.545070:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446664e00 x1874531139549312/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695943.545079:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446664e00 x1874531139549312/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545084:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695943.545085:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.545087:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695943.545088:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695943.545091:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446664e00 x1874531139549312/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545098:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695943.545102:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446664e00 x1874531139549312/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545109:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695943.545111:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.545112:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695943.545114:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.545115:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695943.545118:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695943.545125:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695943.545129:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695943.545131:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695943.545135:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446664e00 x1874531139549312/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695943.545143:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446664e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139549312:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695943.545148:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695943.545152:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446664380 x1874531139549440/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545158:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695943.545159:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695943.545162:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695943.545163:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695943.545165:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695943.545168:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446664380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139549440:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695943.545171:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695943.545173:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695943.545175:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695943.545177:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695943.545180:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445661800. 02000000:00000001:2.0:1787695943.545181:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695943.545185:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695943.545187:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695943.545190:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695943.545193:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139549440, portal 10 00000100:00000001:2.0:1787695943.545195:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695943.545196:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929554816 : -128555779996800 : ffff8b1446664380) 00000100:00000040:2.0:1787695943.545202:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446664380 x1874531139549440/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695968 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545218:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695943.545220:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695943.545223:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695943.545225:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139549440, offset 0 00000400:00000200:2.0:1787695943.545229:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695943.545233:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695943.545237:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695943.545240:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695943.545242:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695943.545245:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695943.545247:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695943.545250:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17102:17102:256:4294967295] 192.168.201.140@tcp LPNI seq info [17101:17101:8:4294967295] 00000400:00000200:2.0:1787695943.545254:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695943.545260:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695943.545264:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695943.545267:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695943.545272:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695943.545274:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1787695943.545278:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695943.545280:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1787695943.545284:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695943.545286:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695943.545289:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695943.545292:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695943.545295:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695943.545297:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695943.545299:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695943.545302:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446664380 x1874531139549440/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695968 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545310:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695943.545313:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446664380 x1874531139549440/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695968 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545331:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695943.545333:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695943.545336:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695943.545339:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:2.0:1787695943.545343:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695943.545345:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298504007 00000100:00000001:2.0:1787695943.545348:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298515258 00000100:00000001:2.0:1787695943.545350:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298504007 00000100:00000001:2.0:1787695943.545352:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298515258 00000100:00000001:2.0:1787695943.545354:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695943.545356:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695943.545358:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695943.545380:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695943/real 1787695943] req@ffff8b1446664380 x1874531139549440/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695943.545388:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695943.545389:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695943.545391:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695943.545394:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695943.545408:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695943.545412:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695943.545416:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446664380 x1874531139549440/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695943.545424:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446664380 x1874531139549440/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545431:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695943.545432:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.545434:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695943.545435:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695943.545439:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446664380 x1874531139549440/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545451:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695943.545455:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446664380 x1874531139549440/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545462:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695943.545466:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.545468:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695943.545471:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.545473:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695943.545477:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695943.545481:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695943.545485:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695943.545488:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695943.545492:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446664380 x1874531139549440/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695943.545500:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446664380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139549440:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695943.545505:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695943.545509:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446666a00 x1874531139549568/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545515:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695943.545516:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695943.545519:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695943.545521:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695943.545523:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695943.545526:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446666a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139549568:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787695943.545530:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695943.545532:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695943.545539:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695943.545541:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695943.545544:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445663c00. 02000000:00000001:2.0:1787695943.545546:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695943.545551:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695943.545553:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695943.545557:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695943.545560:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139549568, portal 4 00000100:00000001:2.0:1787695943.545562:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695943.545563:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929564672 : -128555779986944 : ffff8b1446666a00) 00000100:00000040:2.0:1787695943.545570:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446666a00 x1874531139549568/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695968 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545576:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695943.545579:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695943.545581:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695943.545583:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139549568, offset 0 00000400:00000200:2.0:1787695943.545587:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695943.545591:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695943.545594:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695943.545597:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695943.545599:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695943.545602:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695943.545604:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695943.545608:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17103:17103:256:4294967295] 192.168.201.140@tcp LPNI seq info [17102:17102:8:4294967295] 00000400:00000200:2.0:1787695943.545612:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695943.545618:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695943.545623:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695943.545625:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695943.545629:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695943.545632:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1787695943.545636:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695943.545638:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1787695943.545642:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695943.545648:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695943.545651:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695943.545655:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695943.545658:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695943.545660:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695943.545662:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695943.545666:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446666a00 x1874531139549568/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695968 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545673:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695943.545676:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446666a00 x1874531139549568/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695968 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545683:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695943.545684:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695943.545687:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695943.545691:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:2.0:1787695943.545694:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695943.545696:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298504038 00000100:00000001:2.0:1787695943.545699:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298515258 00000100:00000001:2.0:1787695943.545701:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298504038 00000100:00000001:2.0:1787695943.545703:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298515258 00000100:00000001:2.0:1787695943.545705:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695943.545707:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695943.545709:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695943.545712:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695943/real 1787695943] req@ffff8b1446666a00 x1874531139549568/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695943.545720:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695943.545721:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695943.545723:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695943.545725:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695943.545727:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695943.545736:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695943.545740:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446666a00 x1874531139549568/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695943.545748:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446666a00 x1874531139549568/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545754:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695943.545755:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.545757:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695943.545758:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695943.545762:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446666a00 x1874531139549568/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545767:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695943.545771:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446666a00 x1874531139549568/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545778:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695943.545779:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.545781:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695943.545782:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.545784:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695943.545785:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695943.545787:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695943.545790:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695943.545792:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695943.545795:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446666a00 x1874531139549568/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695943.545803:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446666a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139549568:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787695943.545807:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695943.545810:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665c00 x1874531139549696/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545822:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695943.545824:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695943.545827:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695943.545829:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695943.545831:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695943.545834:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139549696:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787695943.545837:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695943.545839:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695943.545841:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695943.545842:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695943.545846:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445662c00. 02000000:00000001:2.0:1787695943.545848:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695943.545851:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695943.545854:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695943.545857:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695943.545860:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139549696, portal 4 00000100:00000001:2.0:1787695943.545861:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695943.545863:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929561088 : -128555779990528 : ffff8b1446665c00) 00000100:00000040:2.0:1787695943.545869:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665c00 x1874531139549696/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695968 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545875:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695943.545878:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695943.545880:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695943.545882:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139549696, offset 0 00000400:00000200:2.0:1787695943.545886:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695943.545889:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695943.545893:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695943.545896:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695943.545898:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695943.545901:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695943.545903:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695943.545911:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17104:17104:256:4294967295] 192.168.201.140@tcp LPNI seq info [17103:17103:8:4294967295] 00000400:00000200:2.0:1787695943.545916:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695943.545923:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695943.545928:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695943.545931:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695943.545935:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695943.545938:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1787695943.545941:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695943.545944:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1787695943.545947:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695943.545949:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695943.545952:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695943.545956:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695943.545959:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695943.545960:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695943.545963:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695943.545967:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446665c00 x1874531139549696/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695968 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545974:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695943.545978:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665c00 x1874531139549696/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695968 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.545984:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695943.545986:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695943.545989:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695943.545991:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:2.0:1787695943.545994:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695943.545996:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298503710 00000100:00000001:2.0:1787695943.545999:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298515258 00000100:00000001:2.0:1787695943.546001:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298503710 00000100:00000001:2.0:1787695943.546003:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298515258 00000100:00000001:2.0:1787695943.546005:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695943.546011:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695943.546013:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695943.546017:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695943/real 1787695943] req@ffff8b1446665c00 x1874531139549696/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695943.546026:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695943.546027:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695943.546029:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695943.546031:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695943.546033:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695943.546035:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695943.546038:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665c00 x1874531139549696/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695943.546046:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665c00 x1874531139549696/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.546052:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695943.546053:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.546055:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695943.546056:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695943.546059:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446665c00 x1874531139549696/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.546065:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695943.546069:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665c00 x1874531139549696/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.546077:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695943.546080:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.546081:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695943.546083:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.546085:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695943.546086:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695943.546089:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695943.546097:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695943.546099:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695943.546103:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665c00 x1874531139549696/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695943.546111:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139549696:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787695943.546115:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695943.546119:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446666d80 x1874531139549824/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.546125:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695943.546127:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695943.546129:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695943.546131:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695943.546132:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695943.546135:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446666d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139549824:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695943.546137:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695943.546139:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695943.546141:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695943.546143:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695943.546146:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445663800. 02000000:00000001:2.0:1787695943.546148:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695943.546151:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695943.546154:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695943.546157:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695943.546160:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139549824, portal 10 00000100:00000001:2.0:1787695943.546162:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695943.546163:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929565568 : -128555779986048 : ffff8b1446666d80) 00000100:00000040:2.0:1787695943.546170:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446666d80 x1874531139549824/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695968 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.546177:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695943.546184:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695943.546186:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695943.546189:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139549824, offset 0 00000400:00000200:2.0:1787695943.546193:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695943.546197:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695943.546200:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695943.546203:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695943.546206:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695943.546208:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695943.546210:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695943.546213:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17105:17105:256:4294967295] 192.168.201.140@tcp LPNI seq info [17104:17104:8:4294967295] 00000400:00000200:2.0:1787695943.546217:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695943.546223:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695943.546227:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695943.546230:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695943.546234:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695943.546237:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1787695943.546240:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695943.546243:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1787695943.546246:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695943.546248:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695943.546251:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695943.546254:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695943.546257:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695943.546258:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695943.546261:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695943.546264:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446666d80 x1874531139549824/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695968 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.546272:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695943.546275:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446666d80 x1874531139549824/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695968 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.546281:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695943.546283:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695943.546290:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695943.546293:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:2.0:1787695943.546296:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695943.546298:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298503896 00000100:00000001:2.0:1787695943.546301:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298515258 00000100:00000001:2.0:1787695943.546303:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298503896 00000100:00000001:2.0:1787695943.546305:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298515258 00000100:00000001:2.0:1787695943.546307:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695943.546309:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695943.546311:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695943.546316:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695943/real 1787695943] req@ffff8b1446666d80 x1874531139549824/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695943.546324:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695943.546326:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695943.546328:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695943.546330:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695943.546332:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695943.546334:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695943.546337:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446666d80 x1874531139549824/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695943.546345:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446666d80 x1874531139549824/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.546351:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695943.546352:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.546354:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695943.546355:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695943.546359:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446666d80 x1874531139549824/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.546387:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695943.546394:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446666d80 x1874531139549824/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.546424:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695943.546427:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.546429:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695943.546431:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.546433:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695943.546435:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695943.546438:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695943.546443:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695943.546447:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695943.546452:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446666d80 x1874531139549824/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695943.546461:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446666d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139549824:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695943.546467:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.546470:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695943.546475:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446666300 x1874531139549184/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.546483:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695943.546486:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695943.546490:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445660000. 02000000:00000001:2.0:1787695943.546494:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695943.546496:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695943.546499:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:2.0:1787695943.546502:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695943.546505:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545c00. 02000000:00000001:2.0:1787695943.546511:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695943.546513:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695943.546515:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446666300. 00000100:00000001:2.0:1787695943.546519:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695943.546520:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.546528:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695943.546534:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446664e00 x1874531139549312/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.546541:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695943.546543:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695943.546546:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445662400. 02000000:00000001:2.0:1787695943.546548:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695943.546550:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695943.546552:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:2.0:1787695943.546554:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695943.546556:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446547400. 02000000:00000001:2.0:1787695943.546560:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695943.546561:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695943.546562:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446664e00. 00000100:00000001:2.0:1787695943.546566:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695943.546567:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.546569:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695943.546573:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446664380 x1874531139549440/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.546580:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695943.546582:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695943.546583:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445661800. 02000000:00000001:2.0:1787695943.546586:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695943.546588:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695943.546589:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:2.0:1787695943.546591:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695943.546593:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446544c00. 02000000:00000001:2.0:1787695943.546597:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695943.546598:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695943.546600:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446664380. 00000100:00000001:2.0:1787695943.546602:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695943.546604:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.546606:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695943.546610:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446666a00 x1874531139549568/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.546628:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695943.546629:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695943.546631:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445663c00. 02000000:00000001:2.0:1787695943.546634:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695943.546636:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695943.546637:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:2.0:1787695943.546639:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695943.546641:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545000. 02000000:00000001:2.0:1787695943.546644:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695943.546646:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695943.546647:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446666a00. 00000100:00000001:2.0:1787695943.546650:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695943.546651:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.546653:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695943.546657:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665c00 x1874531139549696/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.546664:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695943.546666:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695943.546667:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445662c00. 02000000:00000001:2.0:1787695943.546670:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695943.546671:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695943.546672:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:2.0:1787695943.546674:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695943.546676:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545800. 02000000:00000001:2.0:1787695943.546678:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695943.546679:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695943.546680:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665c00. 00000100:00000001:2.0:1787695943.546683:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695943.546684:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.546686:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695943.546690:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446666d80 x1874531139549824/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.546697:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695943.546698:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695943.546706:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445663800. 02000000:00000001:2.0:1787695943.546709:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695943.546711:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695943.546713:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:2.0:1787695943.546715:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695943.546716:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545400. 02000000:00000001:2.0:1787695943.546719:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695943.546720:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695943.546722:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446666d80. 00000100:00000001:2.0:1787695943.546725:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695943.546727:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.546729:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.546741:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695943.546744:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695943.546746:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695943.546748:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695943.546750:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695943.546755:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446667480 x1874531139549952/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.546763:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695943.546765:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695943.546768:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695943.546769:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695943.546772:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695943.546775:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446667480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139549952:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695943.546779:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695943.546781:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695943.546783:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695943.546785:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695943.546788:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445663800. 02000000:00000001:2.0:1787695943.546790:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695943.546794:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695943.546797:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695943.546801:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695943.546804:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139549952, portal 10 00000100:00000001:2.0:1787695943.546813:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695943.546814:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929567360 : -128555779984256 : ffff8b1446667480) 00000100:00000040:2.0:1787695943.546821:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446667480 x1874531139549952/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695968 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.546828:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695943.546831:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695943.546834:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695943.546836:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139549952, offset 0 00000400:00000200:2.0:1787695943.546841:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695943.546846:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695943.546850:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695943.546853:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695943.546856:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695943.546859:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695943.546861:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695943.546864:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17106:17106:256:4294967295] 192.168.201.140@tcp LPNI seq info [17105:17105:8:4294967295] 00000400:00000200:2.0:1787695943.546869:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695943.546876:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695943.546880:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695943.546883:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695943.546888:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695943.546891:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1787695943.546895:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695943.546899:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1787695943.546903:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695943.546905:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695943.546908:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695943.546911:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695943.546914:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695943.546916:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695943.546919:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695943.546922:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446667480 x1874531139549952/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695968 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.546943:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695943.546947:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446667480 x1874531139549952/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695968 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.546954:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695943.546956:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695943.546960:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695943.546964:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:2.0:1787695943.546968:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695943.546970:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298503875 00000100:00000001:2.0:1787695943.546973:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298515258 00000100:00000001:2.0:1787695943.546975:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298503875 00000100:00000001:2.0:1787695943.546977:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298515258 00000100:00000001:2.0:1787695943.546980:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695943.546982:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695943.546984:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695943.546989:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695943/real 1787695943] req@ffff8b1446667480 x1874531139549952/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695943.546998:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695943.546999:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695943.547000:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695943.547002:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695943.547003:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695943.547005:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695943.547008:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446667480 x1874531139549952/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695943.547013:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446667480 x1874531139549952/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.547017:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695943.547023:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.547025:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695943.547026:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695943.547029:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446667480 x1874531139549952/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.547034:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695943.547036:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446667480 x1874531139549952/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.547041:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695943.547043:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.547044:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695943.547045:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.547046:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695943.547048:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695943.547049:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695943.547051:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695943.547052:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695943.547054:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446667480 x1874531139549952/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695943.547059:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446667480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139549952:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695943.547062:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.547063:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695943.547065:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446667480 x1874531139549952/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695968 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695943.547070:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695943.547071:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695943.547072:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445663800. 02000000:00000001:2.0:1787695943.547073:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695943.547074:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695943.547079:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:2.0:1787695943.547080:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695943.547082:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446546800. 02000000:00000001:2.0:1787695943.547085:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695943.547086:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695943.547087:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446667480. 00000100:00000001:2.0:1787695943.547089:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695943.547090:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.547092:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695943.547095:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695943.547096:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695943.547097:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695943.547099:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695943.606453:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695943.606460:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695943.606461:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695943.606466:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695943.606468:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695943.606471:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695943.606473:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695943.606475:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695943.606494:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695944.246526:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695944.246535:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695944.246543:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695944.246544:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695944.246548:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695944.246549:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695944.246553:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695944.246554:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695944.246556:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695944.246557:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695944.246560:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695944.246560:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695944.246646:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695944.246652:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695944.246655:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695944.630497:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695944.630504:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695944.630506:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695944.630510:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695944.630513:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695944.630516:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695944.630518:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695944.630520:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695944.630522:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695945.270503:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695945.270511:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695945.270536:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695945.270538:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695945.270542:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695945.270543:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695945.270548:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695945.270549:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695945.270553:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695945.270554:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695945.270558:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695945.270559:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695945.270624:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695945.270628:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695945.270632:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695945.654422:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695945.654428:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695945.654430:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695945.654434:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695945.654437:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695945.654440:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695945.654442:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695945.654444:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695945.654446:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695946.294491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695946.294498:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695946.294505:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695946.294506:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695946.294510:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695946.294514:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695946.294516:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695946.294523:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695946.294525:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695946.294528:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695946.294531:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695946.294533:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695946.294536:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695946.294538:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695946.294543:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695946.294544:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695946.294546:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695946.294548:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695946.294549:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695946.294550:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695946.294552:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695946.294553:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695946.294555:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695946.294557:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695946.294560:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695946.294562:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695946.294565:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695946.294567:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695946.294568:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695946.294588:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695946.294589:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695946.294591:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695946.294593:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695946.294594:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695946.294596:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695946.294598:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695946.294664:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695946.294667:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695946.294670:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695946.678438:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695946.678444:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695946.678446:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695946.678451:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695946.678455:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695946.678458:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695946.678460:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695946.678462:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695946.678464:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695947.318478:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695947.318484:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695947.318489:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695947.318490:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695947.318492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695947.318493:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695947.318495:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695947.318496:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695947.318498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695947.318499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695947.318501:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695947.318502:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695947.318555:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695947.318558:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695947.318561:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695947.702425:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695947.702431:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695947.702433:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695947.702437:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695947.702440:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695947.702442:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695947.702444:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695947.702467:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695947.702470:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695948.342476:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695948.342482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695948.342486:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695948.342487:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695948.342490:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695948.342490:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695948.342493:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695948.342494:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695948.342496:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695948.342497:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695948.342500:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695948.342501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695948.342567:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695948.342571:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695948.342573:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787695948.662589:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695948.662601:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787695948.662604:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695948.662609:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695948.662614:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695948.662635:0:47074:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3836 00000400:00000200:2.0:1787695948.662643:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695948.662650:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695948.662653:0:47074:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695948.662657:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695948.662662:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695948.662664:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695948.662668:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695948.662674:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695948.662676:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:2.0:1787695948.662680:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695948.662685:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:2.0:1787695948.662689:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:2.0:1787695948.662691:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:2.0:1787695948.662695:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695948.662716:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695948.662724:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695948.662726:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695948.662729:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:2.0:1787695948.662732:0:47074:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695948.662735:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695948.662738:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695948.662741:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695948.662744:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695948.662746:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787695948.662749:0:47074:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695948.662755:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ad500. 00000020:00000040:2.0:1787695948.662759:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:2.0:1787695948.662763:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695948.662765:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695948.662768:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695948.662771:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695948.662775:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695948.662783:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663800. 00000100:00000001:2.0:1787695948.662791:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695948.662801:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ad500 x1874531139550080/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695948.662812:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695948.662817:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695948.662823:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004ab396cc] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ad500 x1874531139550080/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.662876:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695948.662879:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695948.662881:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695948.662887:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ad500 x1874531139550080/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.662893:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695948.662899:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695948.662903:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695948.662905:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695948.662907:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695948.662909:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139550080:192.168.201.140@tcp:250:kworker.0 00000100:00000001:1.0:1787695948.662912:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695948.662916:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695948.662918:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695948.662920:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695948.662923:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695948.662925:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695948.662930:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695948.662937:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695948.662943:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695948.662950:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139550080, portal 25 00000100:00000001:1.0:1787695948.662952:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695948.662953:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928805632 : -128555780745984 : ffff8b14465ad500) 00000100:00000040:1.0:1787695948.662958:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ad500 x1874531139550080/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695973 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.662964:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695948.662966:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695948.662968:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695948.662971:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139550080, offset 0 00000400:00000200:1.0:1787695948.662977:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695948.662985:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695948.662992:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695948.662997:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695948.663002:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695948.663005:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695948.663009:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695948.663013:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17107:17107:256:4294967295] 192.168.201.140@tcp LPNI seq info [17106:17106:8:4294967295] 00000400:00000200:1.0:1787695948.663018:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695948.663029:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695948.663043:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695948.663050:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695948.663057:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695948.663060:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1787695948.663065:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695948.663067:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1787695948.663071:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695948.663073:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695948.663079:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695948.663083:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695948.663087:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695948.663089:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695948.663093:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695948.663097:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ad500 x1874531139550080/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695973 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695948.663127:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695948.663130:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695948.663132:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787695948.663133:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695948.663135:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695948.663137:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695948.663143:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3836 00000400:00000200:2.0:1787695948.663146:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695948.663148:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695948.663150:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695948.663152:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695948.663153:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695948.663155:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695948.663156:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695948.663158:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695948.663159:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:2.0:1787695948.663160:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695948.663161:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:2.0:1787695948.663163:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:2.0:1787695948.663164:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:2.0:1787695948.663166:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695948.663168:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695948.663170:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695948.663172:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695948.663175:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:2.0:1787695948.663178:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695948.663180:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695948.663182:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695948.663188:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695948.663190:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695948.663192:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695948.663197:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787695948.663200:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695948.663202:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695948.663205:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465afb80. 00000020:00000040:2.0:1787695948.663208:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:2.0:1787695948.663211:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695948.663220:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695948.663222:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695948.663224:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695948.663226:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695948.663230:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445662c00. 00000100:00000001:2.0:1787695948.663234:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695948.663241:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465afb80 x1874531139550208/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695948.663249:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695948.663251:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695948.663254:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b9acb62b] to pc [ptlrpcd_rcv+-1] req@ffff8b14465afb80 x1874531139550208/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.663279:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695948.663282:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ad500 x1874531139550080/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695973 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.663287:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.663288:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695948.663294:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695948.663297:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695948.663299:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298520376 00000100:00000001:1.0:1787695948.663305:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298515258 00000100:00000001:1.0:1787695948.663332:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298520376 00000100:00000001:1.0:1787695948.663340:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298515258 00000100:00000001:1.0:1787695948.663344:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695948.663344:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.663346:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.663349:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1787695948.663351:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787695948.663354:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695948/real 1787695948] req@ffff8b14465ad500 x1874531139550080/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695948.663355:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787695948.663356:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695948.663358:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695948.663361:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695948.663410:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3836 00000400:00000001:1.0:1787695948.663412:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695948.663415:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:2.0:1787695948.663415:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695948.663418:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:2.0:1787695948.663419:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695948.663421:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1787695948.663422:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695948.663424:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000040:2.0:1787695948.663424:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695948.663427:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787695948.663427:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695948.663429:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000200:1.0:1787695948.663432:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ad500 x1874531139550080/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695948.663432:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695948.663435:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695948.663436:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:2.0:1787695948.663438:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695948.663440:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000100:00000200:1.0:1787695948.663442:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ad500 x1874531139550080/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787695948.663442:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:2.0:1787695948.663449:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000001:1.0:1787695948.663450:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695948.663452:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695948.663453:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695948.663455:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695948.663455:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695948.663457:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695948.663457:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695948.663459:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00100000:1.0:1787695948.663462:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ad500 x1874531139550080/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1787695948.663462:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:2.0:1787695948.663465:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695948.663467:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695948.663469:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695948.663470:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787695948.663473:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1787695948.663475:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ad500 x1874531139550080/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695948.663475:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695948.663477:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695948.663480:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787695948.663483:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:2.0:1787695948.663483:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695948.663485:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.663486:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.663488:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000010:2.0:1787695948.663488:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ad180. 00000100:00000001:1.0:1787695948.663489:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1787695948.663490:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000100:00000001:1.0:1787695948.663492:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1787695948.663493:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787695948.663494:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1787695948.663494:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695948.663496:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00080000:1.0:1787695948.663497:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1787695948.663498:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695948.663500:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695948.663501:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695948.663504:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000010:2.0:1787695948.663504:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663c00. 00000100:00000001:2.0:1787695948.663507:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695948.663508:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ad500 x1874531139550080/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695948.663512:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ad180 x1874531139550336/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695948.663520:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1787695948.663523:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139550080:192.168.201.140@tcp:250:kworker.0 00000100:00000001:2.0:1787695948.663523:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695948.663527:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006ad773a4] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ad180 x1874531139550336/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.663529:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.663531:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695948.663535:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ad500 x1874531139550080/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695948.663535:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695948.663541:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695948.663542:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695948.663544:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:2.0:1787695948.663544:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000010:1.0:1787695948.663546:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547c00. 00000100:00000001:2.0:1787695948.663546:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695948.663548:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1787695948.663550:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787695948.663550:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000020:00000001:1.0:1787695948.663552:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695948.663553:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000100:00080000:2.0:1787695948.663553:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3836 00000020:00000001:1.0:1787695948.663556:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000400:00000200:2.0:1787695948.663557:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000010:1.0:1787695948.663558:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663800. 00000400:00000200:2.0:1787695948.663560:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 02000000:00000001:1.0:1787695948.663562:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:2.0:1787695948.663562:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1787695948.663564:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695948.663565:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ad500. 00000100:00000040:2.0:1787695948.663565:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695948.663567:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695948.663568:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000040:2.0:1787695948.663569:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695948.663570:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.663572:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695948.663572:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695948.663574:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695948.663575:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:2.0:1787695948.663577:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695948.663579:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:2.0:1787695948.663579:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000100:00000001:1.0:1787695948.663581:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787695948.663581:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:2.0:1787695948.663583:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:1.0:1787695948.663584:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695948.663585:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1787695948.663586:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695948.663587:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787695948.663588:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695948.663591:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465afb80 x1874531139550208/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695948.663591:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695948.663593:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695948.663595:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:2.0:1787695948.663598:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695948.663599:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695948.663600:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695948.663601:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695948.663603:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695948.663603:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695948.663605:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695948.663605:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695948.663607:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695948.663607:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695948.663609:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:1.0:1787695948.663610:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139550208:192.168.201.140@tcp:38:kworker.0 00000008:00100000:2.0:1787695948.663615:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787695948.663617:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:2.0:1787695948.663618:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695948.663619:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787695948.663620:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695948.663621:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695948.663623:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:2.0:1787695948.663623:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465adc00. 02000000:00000010:1.0:1787695948.663625:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547c00. 00000020:00000040:2.0:1787695948.663625:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:1.0:1787695948.663627:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695948.663628:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695948.663629:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695948.663631:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695948.663631:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695948.663633:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1787695948.663634:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 02000000:00000001:2.0:1787695948.663634:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695948.663638:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 02000000:00000010:2.0:1787695948.663638:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661800. 00000100:00000200:1.0:1787695948.663642:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139550208, portal 10 00000100:00000001:2.0:1787695948.663642:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.663644:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695948.663645:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928815488 : -128555780736128 : ffff8b14465afb80) 00000100:00100000:2.0:1787695948.663647:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465adc00 x1874531139550464/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1787695948.663651:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465afb80 x1874531139550208/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695973 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695948.663654:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695948.663656:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.663659:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695948.663661:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000040:2.0:1787695948.663661:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000114811d4] to pc [ptlrpcd_rcv+-1] req@ffff8b14465adc00 x1874531139550464/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1787695948.663664:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695948.663666:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139550208, offset 0 00000100:00000001:2.0:1787695948.663670:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695948.663671:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00080000:2.0:1787695948.663673:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695948.663675:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:2.0:1787695948.663676:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:1.0:1787695948.663678:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:2.0:1787695948.663678:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695948.663680:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1787695948.663682:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1787695948.663682:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695948.663684:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3836 00000400:00000200:1.0:1787695948.663685:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695948.663687:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695948.663688:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695948.663690:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695948.663691:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000200:1.0:1787695948.663693:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17108:17108:256:4294967295] 192.168.201.140@tcp LPNI seq info [17107:17107:8:4294967295] 00000100:00080000:2.0:1787695948.663693:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695948.663696:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695948.663698:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695948.663699:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695948.663701:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695948.663704:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:2.0:1787695948.663708:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000200:1.0:1787695948.663709:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000020:00000001:2.0:1787695948.663710:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695948.663712:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000800:00000010:1.0:1787695948.663713:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000020:00000001:2.0:1787695948.663714:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695948.663716:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000800:00000200:1.0:1787695948.663717:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000020:00000001:2.0:1787695948.663718:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:2.0:1787695948.663720:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000800:00000200:1.0:1787695948.663724:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000100:00000040:2.0:1787695948.663724:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695948.663726:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695948.663728:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000200:1.1:1787695948.663729:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:2.0:1787695948.663730:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000200:1.1:1787695948.663732:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000020:00000040:2.0:1787695948.663733:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:2.0:1787695948.663736:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000800:00000100:1.0:1787695948.663737:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695948.663739:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000100:00000001:2.0:1787695948.663739:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695948.663741:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1787695948.663742:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:2.0:1787695948.663744:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1787695948.663746:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 02000000:00000001:2.0:1787695948.663746:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695948.663748:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:1.0:1787695948.663749:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000008:00100000:2.0:1787695948.663750:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000400:00000010:1.0:1787695948.663751:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000008:00000001:2.0:1787695948.663753:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.663754:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787695948.663755:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695948.663758:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465afb80 x1874531139550208/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695973 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1787695948.663759:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ae300. 00000020:00000040:2.0:1787695948.663761:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:2.0:1787695948.663764:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695948.663765:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.663767:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787695948.663767:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695948.663769:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1787695948.663770:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465afb80 x1874531139550208/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695973 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695948.663771:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695948.663774:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445662400. 00000100:00000001:2.0:1787695948.663777:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.663778:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.663779:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695948.663782:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695948.663782:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ae300 x1874531139550592/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1787695948.663786:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000400:00000200:1.0:1787695948.663789:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695948.663792:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298510278 00000100:00000001:1.0:1787695948.663795:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298520376 00000100:00000001:1.0:1787695948.663797:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298510278 00000100:00000001:1.0:1787695948.663800:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298520376 00000100:00000001:2.0:1787695948.663800:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695948.663802:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695948.663802:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.663804:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.663806:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:2.0:1787695948.663807:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004515329a] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ae300 x1874531139550592/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695948.663810:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695948/real 1787695948] req@ffff8b14465afb80 x1874531139550208/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695948.663815:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695948.663818:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:1.0:1787695948.663819:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695948.663820:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:2.0:1787695948.663822:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787695948.663824:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695948.663825:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:1.0:1787695948.663827:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000100:00000001:2.0:1787695948.663828:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1787695948.663829:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695948.663831:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00080000:2.0:1787695948.663831:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3836 00000100:00000001:1.0:1787695948.663834:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:2.0:1787695948.663835:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000200:1.0:1787695948.663838:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465afb80 x1874531139550208/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787695948.663838:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695948.663840:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695948.663843:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695948.663846:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1787695948.663847:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465afb80 x1874531139550208/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695948.663848:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695948.663851:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695948.663853:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695948.663855:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000040:2.0:1787695948.663855:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:2.0:1787695948.663856:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695948.663857:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.663859:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:2.0:1787695948.663859:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000100:00000001:1.0:1787695948.663860:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787695948.663860:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000020:00000001:2.0:1787695948.663863:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00100000:1.0:1787695948.663864:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465afb80 x1874531139550208/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695948.663866:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695948.663868:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695948.663870:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695948.663872:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695948.663873:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000040:1.0:1787695948.663876:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465afb80 x1874531139550208/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1787695948.663878:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:2.0:1787695948.663882:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695948.663884:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695948.663884:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695948.663886:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695948.663887:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.663888:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1787695948.663889:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695948.663890:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.663891:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1787695948.663891:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.663893:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695948.663893:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695948.663895:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1787695948.663896:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00000001:2.0:1787695948.663898:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695948.663899:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695948.663900:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.663902:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000010:2.0:1787695948.663903:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465aed80. 00000020:00000040:2.0:1787695948.663905:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000100:00000040:1.0:1787695948.663906:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465afb80 x1874531139550208/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695948.663908:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695948.663910:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695948.663912:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695948.663913:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1787695948.663915:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139550208:192.168.201.140@tcp:38:kworker.0 02000000:00000001:2.0:1787695948.663915:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695948.663918:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445660000. 00000100:00000001:2.0:1787695948.663921:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.663922:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695948.663926:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ad180 x1874531139550336/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695948.663926:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465aed80 x1874531139550720/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695948.663934:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695948.663934:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695948.663936:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695948.663936:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695948.663938:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695948.663940:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1787695948.663940:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bed91c58] to pc [ptlrpcd_rcv+-1] req@ffff8b14465aed80 x1874531139550720/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695948.663942:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695948.663945:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139550336:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695948.663949:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787695948.663949:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695948.663951:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1787695948.663952:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1787695948.663953:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695948.663954:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1787695948.663955:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000010:1.0:1787695948.663957:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 00000100:00000001:2.0:1787695948.663957:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695948.663958:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1787695948.663959:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695948.663961:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1787695948.663963:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695948.663965:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000100:00080000:2.0:1787695948.663968:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3836 00000400:00000010:1.0:1787695948.663969:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695948.663972:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139550336, portal 10 00000400:00000200:2.0:1787695948.663972:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695948.663974:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695948.663975:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928804736 : -128555780746880 : ffff8b14465ad180) 00000100:00000040:1.0:1787695948.663981:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ad180 x1874531139550336/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695973 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787695948.663984:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695948.663987:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695948.663988:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695948.663990:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695948.663991:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695948.663992:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1787695948.663993:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000040:2.0:1787695948.663994:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000200:1.0:1787695948.663996:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139550336, offset 0 00000100:00000001:2.0:1787695948.663997:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695948.663999:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:1.0:1787695948.664000:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000020:00000040:2.0:1787695948.664001:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000020:00000001:2.0:1787695948.664002:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:1.0:1787695948.664004:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000040:2.0:1787695948.664004:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:2.0:1787695948.664006:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000400:00000200:1.0:1787695948.664007:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000020:00000001:2.0:1787695948.664009:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000400:00000200:1.0:1787695948.664010:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1787695948.664012:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695948.664014:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695948.664015:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695948.664017:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:2.0:1787695948.664018:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695948.664020:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000020:00000040:2.0:1787695948.664020:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000400:00000200:1.0:1787695948.664022:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00080000:2.0:1787695948.664023:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:1.0:1787695948.664026:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17109:17109:256:4294967295] 192.168.201.140@tcp LPNI seq info [17108:17108:8:4294967295] 00000100:00000001:2.0:1787695948.664026:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695948.664028:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1787695948.664031:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:2.0:1787695948.664031:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695948.664033:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695948.664035:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:1.0:1787695948.664038:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000008:00100000:2.0:1787695948.664038:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787695948.664040:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695948.664042:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695948.664043:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000010:2.0:1787695948.664045:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465aea00. 00000800:00000010:1.0:1787695948.664046:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000020:00000040:2.0:1787695948.664048:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000800:00000200:1.0:1787695948.664050:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:2.0:1787695948.664050:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000200:1.0:1787695948.664053:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 02000000:00000001:2.0:1787695948.664055:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695948.664057:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.1:1787695948.664057:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:2.0:1787695948.664059:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:1.1:1787695948.664060:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 02000000:00000001:2.0:1787695948.664061:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787695948.664064:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 02000000:00000010:2.0:1787695948.664065:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661c00. 00000800:00000010:1.0:1787695948.664066:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000100:00000001:2.0:1787695948.664068:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695948.664070:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00100000:2.0:1787695948.664073:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465aea00 x1874531139550848/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695948.664074:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695948.664077:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695948.664079:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695948.664081:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787695948.664081:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695948.664083:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695948.664085:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ad180 x1874531139550336/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695973 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695948.664087:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a41ef0cb] to pc [ptlrpcd_rcv+-1] req@ffff8b14465aea00 x1874531139550848/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664094:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787695948.664096:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695948.664097:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ad180 x1874531139550336/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695973 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695948.664098:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1787695948.664102:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1787695948.664105:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695948.664105:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695948.664107:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:2.0:1787695948.664109:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1787695948.664110:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695948.664112:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000100:00000040:2.0:1787695948.664114:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3846) 00000400:00000200:1.0:1787695948.664116:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695948.664118:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298509126 00000100:00000001:1.0:1787695948.664120:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298520376 00000100:00000001:1.0:1787695948.664122:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298509126 00000100:00000001:1.0:1787695948.664129:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298520376 00000100:00000001:1.0:1787695948.664131:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695948.664133:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.664136:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695948.664140:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695948/real 1787695948] req@ffff8b14465ad180 x1874531139550336/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695948.664149:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695948.664151:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695948.664152:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695948.664155:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695948.664157:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695948.664159:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695948.664163:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ad180 x1874531139550336/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695948.664172:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ad180 x1874531139550336/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664179:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695948.664181:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.664182:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695948.664183:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695948.664186:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ad180 x1874531139550336/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664191:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695948.664193:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ad180 x1874531139550336/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664197:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695948.664199:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.664200:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695948.664201:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.664206:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695948.664207:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695948.664209:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695948.664212:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695948.664213:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695948.664216:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ad180 x1874531139550336/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695948.664221:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139550336:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695948.664223:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.664225:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695948.664227:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465afb80 x1874531139550208/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664232:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695948.664233:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695948.664234:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695948.664236:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695948.664237:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695948.664238:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695948.664239:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695948.664241:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445662c00. 02000000:00000001:1.0:1787695948.664242:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695948.664243:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695948.664244:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465afb80. 00000100:00000001:1.0:1787695948.664247:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695948.664247:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.664248:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695948.664250:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ad180 x1874531139550336/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664254:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695948.664255:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695948.664256:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695948.664260:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695948.664261:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695948.664262:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695948.664263:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695948.664264:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663c00. 02000000:00000001:1.0:1787695948.664266:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695948.664266:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695948.664267:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ad180. 00000100:00000001:1.0:1787695948.664269:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695948.664270:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.664272:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.664276:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695948.664277:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.664279:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695948.664280:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695948.664282:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695948.664286:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465adc00 x1874531139550464/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664293:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695948.664295:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695948.664297:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695948.664299:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695948.664301:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695948.664304:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139550464:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695948.664318:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695948.664321:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695948.664322:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695948.664324:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695948.664327:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695948.664330:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695948.664334:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695948.664337:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695948.664341:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695948.664344:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139550464, portal 4 00000100:00000001:1.0:1787695948.664347:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695948.664353:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928807424 : -128555780744192 : ffff8b14465adc00) 00000100:00000040:1.0:1787695948.664361:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465adc00 x1874531139550464/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695973 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664450:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695948.664453:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695948.664456:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695948.664459:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139550464, offset 0 00000400:00000200:1.0:1787695948.664462:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695948.664472:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695948.664476:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695948.664478:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695948.664483:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695948.664485:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695948.664486:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695948.664489:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17110:17110:256:4294967295] 192.168.201.140@tcp LPNI seq info [17109:17109:8:4294967295] 00000400:00000200:1.0:1787695948.664492:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695948.664496:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695948.664500:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695948.664502:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695948.664505:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695948.664508:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1787695948.664511:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695948.664513:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1787695948.664516:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695948.664517:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695948.664520:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695948.664522:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695948.664524:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695948.664526:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695948.664528:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695948.664531:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465adc00 x1874531139550464/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695973 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664538:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695948.664546:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465adc00 x1874531139550464/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695973 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664551:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.664553:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695948.664556:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695948.664558:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695948.664561:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695948.664563:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298509157 00000100:00000001:1.0:1787695948.664564:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298520376 00000100:00000001:1.0:1787695948.664566:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298509157 00000100:00000001:1.0:1787695948.664568:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298520376 00000100:00000001:1.0:1787695948.664569:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695948.664570:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.664572:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695948.664574:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695948/real 1787695948] req@ffff8b14465adc00 x1874531139550464/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695948.664580:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695948.664581:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695948.664582:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695948.664584:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695948.664585:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695948.664587:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695948.664589:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465adc00 x1874531139550464/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695948.664595:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465adc00 x1874531139550464/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664599:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695948.664600:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.664601:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695948.664602:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695948.664608:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465adc00 x1874531139550464/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664613:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695948.664616:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465adc00 x1874531139550464/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664620:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695948.664622:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.664623:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695948.664624:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.664625:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695948.664627:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695948.664629:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695948.664631:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695948.664633:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695948.664635:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465adc00 x1874531139550464/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695948.664640:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139550464:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695948.664643:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695948.664645:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ae300 x1874531139550592/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664650:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695948.664651:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695948.664652:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695948.664653:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695948.664655:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695948.664657:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139550592:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695948.664658:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695948.664663:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695948.664664:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695948.664665:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695948.664667:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695948.664669:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695948.664671:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695948.664673:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695948.664675:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695948.664677:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139550592, portal 4 00000100:00000001:1.0:1787695948.664679:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695948.664680:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928809216 : -128555780742400 : ffff8b14465ae300) 00000100:00000040:1.0:1787695948.664684:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ae300 x1874531139550592/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695973 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664689:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695948.664690:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695948.664692:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695948.664693:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139550592, offset 0 00000400:00000200:1.0:1787695948.664696:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695948.664698:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695948.664701:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695948.664703:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695948.664704:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695948.664706:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695948.664708:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695948.664710:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17111:17111:256:4294967295] 192.168.201.140@tcp LPNI seq info [17110:17110:8:4294967295] 00000400:00000200:1.0:1787695948.664713:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695948.664717:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695948.664720:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695948.664722:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695948.664725:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695948.664727:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1787695948.664729:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695948.664731:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1787695948.664734:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695948.664738:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695948.664740:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695948.664743:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695948.664745:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695948.664746:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695948.664748:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695948.664750:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ae300 x1874531139550592/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695973 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664756:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695948.664758:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ae300 x1874531139550592/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695973 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664762:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.664763:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695948.664765:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695948.664767:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695948.664769:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695948.664771:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298508829 00000100:00000001:1.0:1787695948.664772:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298520376 00000100:00000001:1.0:1787695948.664774:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298508829 00000100:00000001:1.0:1787695948.664775:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298520376 00000100:00000001:1.0:1787695948.664777:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695948.664778:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.664779:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695948.664781:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695948/real 1787695948] req@ffff8b14465ae300 x1874531139550592/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695948.664786:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695948.664787:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695948.664788:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695948.664789:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695948.664790:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695948.664795:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695948.664797:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ae300 x1874531139550592/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695948.664804:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ae300 x1874531139550592/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664811:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695948.664813:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.664815:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695948.664817:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695948.664820:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ae300 x1874531139550592/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664827:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695948.664832:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ae300 x1874531139550592/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664839:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695948.664842:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.664844:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695948.664845:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.664847:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695948.664849:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695948.664852:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695948.664855:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695948.664857:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695948.664860:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ae300 x1874531139550592/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695948.664866:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139550592:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695948.664869:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695948.664871:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465aed80 x1874531139550720/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664880:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695948.664882:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695948.664883:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695948.664885:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695948.664886:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695948.664888:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139550720:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695948.664891:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695948.664892:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695948.664893:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695948.664894:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695948.664896:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695948.664898:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695948.664900:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695948.664902:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695948.664904:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695948.664907:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139550720, portal 10 00000100:00000001:1.0:1787695948.664908:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695948.664909:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928811904 : -128555780739712 : ffff8b14465aed80) 00000100:00000040:1.0:1787695948.664913:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465aed80 x1874531139550720/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695973 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664918:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695948.664919:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695948.664921:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695948.664922:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139550720, offset 0 00000400:00000200:1.0:1787695948.664925:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695948.664928:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695948.664930:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695948.664932:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695948.664934:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695948.664935:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695948.664937:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695948.664942:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17112:17112:256:4294967295] 192.168.201.140@tcp LPNI seq info [17111:17111:8:4294967295] 00000400:00000200:1.0:1787695948.664945:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695948.664951:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695948.664954:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695948.664956:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695948.664958:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695948.664960:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1787695948.664963:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695948.664965:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1787695948.664967:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695948.664969:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695948.664971:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695948.664973:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695948.664976:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695948.664977:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695948.664979:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695948.664981:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465aed80 x1874531139550720/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695973 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664986:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695948.664988:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465aed80 x1874531139550720/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695973 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.664993:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.664994:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695948.664996:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695948.664998:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695948.665000:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695948.665001:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298509015 00000100:00000001:1.0:1787695948.665003:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298520376 00000100:00000001:1.0:1787695948.665005:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298509015 00000100:00000001:1.0:1787695948.665006:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298520376 00000100:00000001:1.0:1787695948.665008:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695948.665013:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.665014:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695948.665017:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695948/real 1787695948] req@ffff8b14465aed80 x1874531139550720/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695948.665023:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695948.665024:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695948.665025:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695948.665026:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695948.665027:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695948.665029:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695948.665031:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465aed80 x1874531139550720/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695948.665037:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465aed80 x1874531139550720/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.665044:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695948.665045:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.665047:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695948.665048:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695948.665052:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465aed80 x1874531139550720/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.665060:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695948.665065:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465aed80 x1874531139550720/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.665072:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695948.665075:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.665077:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695948.665078:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.665080:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695948.665082:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695948.665085:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695948.665092:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695948.665093:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695948.665096:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465aed80 x1874531139550720/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695948.665102:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139550720:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695948.665104:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695948.665106:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465aea00 x1874531139550848/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.665110:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695948.665111:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695948.665113:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695948.665114:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695948.665115:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695948.665117:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139550848:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695948.665119:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695948.665120:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695948.665121:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695948.665122:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695948.665124:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695948.665126:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695948.665128:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695948.665130:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695948.665132:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695948.665134:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139550848, portal 10 00000100:00000001:1.0:1787695948.665136:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695948.665137:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928811008 : -128555780740608 : ffff8b14465aea00) 00000100:00000040:1.0:1787695948.665140:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465aea00 x1874531139550848/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695973 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.665145:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695948.665149:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695948.665151:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695948.665152:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139550848, offset 0 00000400:00000200:1.0:1787695948.665155:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695948.665158:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695948.665160:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695948.665162:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695948.665164:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695948.665166:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695948.665168:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695948.665170:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17113:17113:256:4294967295] 192.168.201.140@tcp LPNI seq info [17112:17112:8:4294967295] 00000400:00000200:1.0:1787695948.665174:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695948.665180:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695948.665185:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695948.665188:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695948.665192:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695948.665195:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000800:00000200:1.1:1787695948.665200:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695948.665202:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 53 secs later) 00000800:00000100:1.0:1787695948.665206:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695948.665208:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695948.665211:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695948.665215:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695948.665218:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695948.665220:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695948.665222:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695948.665225:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465aea00 x1874531139550848/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695973 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.665230:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695948.665232:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465aea00 x1874531139550848/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695973 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.665237:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.665238:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695948.665245:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695948.665247:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695948.665250:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695948.665251:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298508993 00000100:00000001:1.0:1787695948.665253:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298520376 00000100:00000001:1.0:1787695948.665254:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298508993 00000100:00000001:1.0:1787695948.665256:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298520376 00000100:00000001:1.0:1787695948.665257:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695948.665259:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.665260:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695948.665263:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695948/real 1787695948] req@ffff8b14465aea00 x1874531139550848/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695948.665268:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695948.665269:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695948.665270:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695948.665272:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695948.665273:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695948.665274:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695948.665276:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465aea00 x1874531139550848/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695948.665282:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465aea00 x1874531139550848/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.665285:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695948.665286:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.665287:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695948.665288:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695948.665290:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465aea00 x1874531139550848/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.665294:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695948.665296:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465aea00 x1874531139550848/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.665304:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695948.665305:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.665321:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695948.665323:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.665325:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695948.665327:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695948.665330:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695948.665334:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695948.665337:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695948.665341:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465aea00 x1874531139550848/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695948.665350:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139550848:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695948.665355:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.665357:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695948.665360:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465adc00 x1874531139550464/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.665386:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695948.665388:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695948.665391:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695948.665395:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695948.665396:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695948.665398:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695948.665400:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695948.665402:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661800. 02000000:00000001:1.0:1787695948.665405:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695948.665406:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695948.665407:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465adc00. 00000100:00000001:1.0:1787695948.665410:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695948.665412:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.665419:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695948.665424:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ae300 x1874531139550592/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.665431:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695948.665432:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695948.665434:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695948.665436:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695948.665438:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695948.665439:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695948.665440:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695948.665441:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445662400. 02000000:00000001:1.0:1787695948.665443:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695948.665444:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695948.665445:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ae300. 00000100:00000001:1.0:1787695948.665446:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695948.665447:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.665448:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695948.665451:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465aed80 x1874531139550720/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.665455:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695948.665456:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695948.665457:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695948.665460:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695948.665461:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695948.665461:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695948.665462:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695948.665463:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445660000. 02000000:00000001:1.0:1787695948.665464:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695948.665465:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695948.665466:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465aed80. 00000100:00000001:1.0:1787695948.665467:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695948.665468:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.665469:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695948.665471:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465aea00 x1874531139550848/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695973 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695948.665482:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695948.665483:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695948.665484:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695948.665486:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695948.665486:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695948.665487:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695948.665488:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695948.665489:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661c00. 02000000:00000001:1.0:1787695948.665490:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695948.665491:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695948.665492:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465aea00. 00000100:00000001:1.0:1787695948.665493:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695948.665494:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.665495:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695948.665498:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695948.665498:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695948.665500:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695948.665501:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695948.726512:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695948.726521:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695948.726523:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695948.726528:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695948.726531:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695948.726535:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695948.726538:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695948.726540:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695948.726543:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695949.366457:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695949.366463:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695949.366468:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695949.366469:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695949.366471:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695949.366472:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695949.366475:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695949.366476:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695949.366478:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695949.366479:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695949.366481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695949.366482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695949.366542:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695949.366548:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695949.366552:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695949.750497:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695949.750506:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695949.750509:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695949.750514:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695949.750517:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695949.750521:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695949.750523:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695949.750525:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695949.750528:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695950.390468:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695950.390474:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695950.390478:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695950.390479:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695950.390481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695950.390482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695950.390502:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695950.390503:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695950.390505:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695950.390506:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695950.390508:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695950.390509:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695950.390562:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695950.390565:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695950.390567:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695950.774468:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695950.774474:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695950.774476:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695950.774480:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695950.774483:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695950.774486:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695950.774488:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695950.774490:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695950.774492:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695951.414447:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695951.414454:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695951.414460:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695951.414462:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695951.414465:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695951.414466:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695951.414469:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695951.414470:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695951.414472:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695951.414474:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695951.414477:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695951.414481:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695951.414483:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695951.414484:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695951.414486:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695951.414489:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695951.414490:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695951.414492:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695951.414494:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695951.414495:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695951.414547:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695951.414551:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695951.414553:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695951.798466:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695951.798471:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695951.798473:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695951.798477:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695951.798480:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695951.798483:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695951.798501:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695951.798503:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695951.798505:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695952.438481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695952.438486:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695952.438490:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695952.438491:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695952.438494:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695952.438495:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695952.438497:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695952.438498:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695952.438500:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695952.438501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695952.438504:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695952.438505:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695952.438571:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695952.438575:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695952.438577:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695952.822457:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695952.822464:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695952.822465:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695952.822470:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695952.822472:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695952.822476:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695952.822477:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695952.822479:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695952.822481:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695953.462433:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695953.462441:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695953.462444:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695953.462446:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695953.462451:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695953.462452:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695953.462454:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695953.462455:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695953.462458:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695953.462459:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695953.462461:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695953.462462:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695953.462463:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695953.462464:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695953.462465:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695953.462466:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695953.462469:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695953.462469:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695953.462472:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695953.462473:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695953.462475:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695953.462475:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695953.462478:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695953.462478:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695953.462533:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695953.462538:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695953.462541:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787695953.782617:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695953.782628:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787695953.782631:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695953.782635:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695953.782640:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695953.782647:0:47074:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3841 00000400:00000200:2.0:1787695953.782657:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695953.782664:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695953.782668:0:47074:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695953.782671:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695953.782675:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695953.782695:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695953.782697:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695953.782700:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695953.782702:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:2.0:1787695953.782704:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695953.782707:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:2.0:1787695953.782709:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:2.0:1787695953.782710:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:2.0:1787695953.782713:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695953.782714:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695953.782715:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695953.782717:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695953.782719:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:2.0:1787695953.782721:0:47074:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695953.782723:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695953.782725:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695953.782728:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695953.782730:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695953.782731:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787695953.782733:0:47074:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695953.782738:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465adf80. 00000020:00000040:2.0:1787695953.782740:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:2.0:1787695953.782743:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695953.782744:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695953.782747:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695953.782749:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695953.782752:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695953.782758:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445660800. 00000100:00000001:2.0:1787695953.782763:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695953.782771:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465adf80 x1874531139550976/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695953.782779:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695953.782787:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695953.782791:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ee5f8f2] to pc [ptlrpcd_rcv+-1] req@ffff8b14465adf80 x1874531139550976/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695953.782812:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695953.782816:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695953.782819:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787695953.782825:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695953.782827:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695953.782830:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695953.782832:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3841 00000400:00000200:2.0:1787695953.782837:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695953.782842:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695953.782845:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695953.782848:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695953.782851:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695953.782853:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695953.782856:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695953.782857:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695953.782863:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695953.782867:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695953.782875:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465adf80 x1874531139550976/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.782884:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695953.782886:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695953.782890:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695953.782893:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695953.782895:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695953.782899:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139550976:192.168.201.140@tcp:250:kworker.0 00000100:00000001:1.0:1787695953.782912:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695953.782925:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695953.782927:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695953.782930:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695953.782934:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695953.782936:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695953.782941:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695953.782946:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000020:00000001:2.0:1787695953.782952:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695953.782954:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:2.0:1787695953.782955:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000010:1.0:1787695953.782956:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000020:00000040:2.0:1787695953.782958:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:2.0:1787695953.782962:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:2.0:1787695953.782964:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000200:1.0:1787695953.782966:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139550976, portal 25 00000100:00000040:2.0:1787695953.782968:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695953.782972:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695953.782972:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695953.782977:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928808320 : -128555780743296 : ffff8b14465adf80) 00000100:00000040:2.0:1787695953.782977:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695953.782982:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695953.782985:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00000040:1.0:1787695953.782989:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465adf80 x1874531139550976/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695978 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695953.782991:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695953.783002:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787695953.783004:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695953.783006:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1787695953.783010:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 02000000:00000001:2.0:1787695953.783013:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:1.0:1787695953.783016:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 02000000:00000001:2.0:1787695953.783017:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695953.783022:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000200:1.0:1787695953.783024:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139550976, offset 0 00000008:00100000:2.0:1787695953.783026:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787695953.783030:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695953.783031:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695953.783036:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000010:2.0:1787695953.783037:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ac000. 00000020:00000040:2.0:1787695953.783040:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:2.0:1787695953.783042:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695953.783043:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695953.783047:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695953.783047:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695953.783051:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695953.783053:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695953.783057:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445662000. 00000400:00000200:1.0:1787695953.783058:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:2.0:1787695953.783063:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695953.783067:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00100000:2.0:1787695953.783071:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ac000 x1874531139551104/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695953.783076:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695953.783084:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1787695953.783085:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695953.783089:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695953.783092:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000040:2.0:1787695953.783096:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a231b497] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ac000 x1874531139551104/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695953.783102:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17114:17114:256:4294967295] 192.168.201.140@tcp LPNI seq info [17113:17113:8:4294967295] 00000100:00000001:2.0:1787695953.783109:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695953.783113:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695953.783114:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:2.0:1787695953.783118:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787695953.783120:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695953.783122:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695953.783124:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695953.783129:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3841 00000400:00000200:1.0:1787695953.783136:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000400:00000200:2.0:1787695953.783145:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000800:00000200:1.0:1787695953.783147:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695953.783155:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000400:00000200:2.0:1787695953.783155:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695953.783160:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695953.783163:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695953.783167:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.0:1787695953.783169:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:2.0:1787695953.783170:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695953.783176:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000200:1.0:1787695953.783179:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 48 secs later) 00000020:00000001:2.0:1787695953.783179:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695953.783180:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:2.0:1787695953.783181:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695953.783182:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:2.0:1787695953.783184:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:2.0:1787695953.783185:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:2.0:1787695953.783187:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695953.783190:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695953.783192:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695953.783193:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695953.783200:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:2.0:1787695953.783203:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695953.783208:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1787695953.783208:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:2.0:1787695953.783212:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:1.1:1787695953.783215:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 48 secs later) 02000000:00000001:2.0:1787695953.783216:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695953.783230:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695953.783237:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695953.783239:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787695953.783241:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000008:00000001:2.0:1787695953.783243:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695953.783244:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695953.783247:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ac380. 00000020:00000040:2.0:1787695953.783253:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000800:00000010:1.0:1787695953.783255:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 02000000:00000001:2.0:1787695953.783255:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695953.783258:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695953.783260:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1787695953.783265:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:2.0:1787695953.783267:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695953.783271:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695953.783276:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 02000000:00000010:2.0:1787695953.783276:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445660400. 00000100:00000001:2.0:1787695953.783282:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695953.783286:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000100:00100000:2.0:1787695953.783289:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ac380 x1874531139551232/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1787695953.783291:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:2.0:1787695953.783299:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695953.783301:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787695953.783302:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695953.783310:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e778d47] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ac380 x1874531139551232/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695953.783313:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465adf80 x1874531139550976/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695978 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695953.783324:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695953.783328:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695953.783335:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787695953.783337:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695953.783339:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695953.783340:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787695953.783341:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695953.783346:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3841 00000100:00000040:1.0:1787695953.783350:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465adf80 x1874531139550976/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695978 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787695953.783354:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695953.783360:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695953.783396:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695953.783403:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695953.783404:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695953.783408:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695953.783409:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695953.783414:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695953.783416:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695953.783418:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695953.783419:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1787695953.783419:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:2.0:1787695953.783423:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695953.783426:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:2.0:1787695953.783429:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000400:00000200:1.0:1787695953.783430:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:2.0:1787695953.783431:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:1.0:1787695953.783436:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298525497 00000100:00000040:2.0:1787695953.783444:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695953.783445:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298520376 00000100:00000001:2.0:1787695953.783446:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695953.783451:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695953.783453:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695953.783455:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00000001:1.0:1787695953.783456:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298525497 00000100:00080000:2.0:1787695953.783459:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695953.783463:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.783465:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298520376 02000000:00000001:2.0:1787695953.783465:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695953.783469:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695953.783471:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695953.783472:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695953.783477:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695953.783479:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787695953.783480:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787695953.783485:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000008:00000001:2.0:1787695953.783486:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695953.783490:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695953.783493:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465af100. 00000100:00100000:1.0:1787695953.783496:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695953/real 1787695953] req@ffff8b14465adf80 x1874531139550976/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1787695953.783497:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:2.0:1787695953.783501:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695953.783503:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695953.783506:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695953.783507:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695953.783509:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695953.783514:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663400. 00000100:00000001:2.0:1787695953.783519:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787695953.783520:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695953.783525:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:2.0:1787695953.783525:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465af100 x1874531139551360/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000010:1.0:1787695953.783529:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695953.783536:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000100:00000001:2.0:1787695953.783536:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695953.783540:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695953.783541:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000040:2.0:1787695953.783545:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000094a5074e] to pc [ptlrpcd_rcv+-1] req@ffff8b14465af100 x1874531139551360/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.783548:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787695953.783555:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695953.783557:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465adf80 x1874531139550976/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695953.783558:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695953.783571:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1787695953.783574:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695953.783577:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695953.783583:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695953.783587:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3841 00000100:00000200:1.0:1787695953.783591:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465adf80 x1874531139550976/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787695953.783593:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695953.783598:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695953.783603:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695953.783609:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695953.783612:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695953.783613:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000040:2.0:1787695953.783615:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695953.783618:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695953.783619:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695953.783624:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695953.783625:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000040:2.0:1787695953.783626:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:2.0:1787695953.783630:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695953.783631:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1787695953.783632:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:2.0:1787695953.783636:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:2.0:1787695953.783638:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00100000:1.0:1787695953.783642:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465adf80 x1874531139550976/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695953.783643:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695953.783646:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695953.783649:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695953.783652:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695953.783656:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:2.0:1787695953.783661:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695953.783666:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695953.783669:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695953.783670:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787695953.783672:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695953.783675:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695953.783678:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000040:1.0:1787695953.783679:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465adf80 x1874531139550976/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787695953.783680:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787695953.783688:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695953.783692:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695953.783696:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465aea00. 00000100:00000001:1.0:1787695953.783698:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000040:2.0:1787695953.783700:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 00000100:00000001:1.0:1787695953.783704:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787695953.783704:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695953.783707:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.783710:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695953.783710:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787695953.783715:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787695953.783715:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695953.783718:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.783721:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000010:2.0:1787695953.783722:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661000. 00000100:00000001:1.0:1787695953.783726:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695953.783727:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695953.783734:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1787695953.783735:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465aea00 x1874531139551488/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695953.783745:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00080000:1.0:1787695953.783746:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695953.783747:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.783754:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695953.783755:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a41ef0cb] to pc [ptlrpcd_rcv+-1] req@ffff8b14465aea00 x1874531139551488/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695953.783765:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695953.783767:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465adf80 x1874531139550976/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695953.783768:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695953.783775:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787695953.783777:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695953.783779:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695953.783785:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695953.783789:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3841 00000100:00100000:1.0:1787695953.783795:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139550976:192.168.201.140@tcp:250:kworker.0 00000400:00000200:2.0:1787695953.783795:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695953.783800:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695953.783804:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695953.783807:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695953.783809:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695953.783813:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787695953.783815:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695953.783821:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465adf80 x1874531139550976/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695953.783822:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695953.783833:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695953.783836:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695953.783838:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:2.0:1787695953.783840:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695953.783843:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000020:00000001:2.0:1787695953.783849:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000020:00000001:2.0:1787695953.783855:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:1.0:1787695953.783857:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695953.783862:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695953.783865:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695953.783870:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695953.783874:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000040:2.0:1787695953.783875:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000020:00000040:1.0:1787695953.783876:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000100:00000001:2.0:1787695953.783878:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:1.0:1787695953.783882:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000040:2.0:1787695953.783883:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695953.783887:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000010:1.0:1787695953.783888:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445660800. 00000020:00000040:2.0:1787695953.783889:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:2.0:1787695953.783896:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000001:1.0:1787695953.783897:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1787695953.783898:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695953.783902:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695953.783903:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1787695953.783905:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:1.0:1787695953.783907:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465adf80. 02000000:00000001:2.0:1787695953.783908:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695953.783913:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695953.783915:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000008:00100000:2.0:1787695953.783916:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787695953.783920:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1787695953.783920:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695953.783927:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.783928:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:2.0:1787695953.783930:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465aed80. 00000020:00000040:2.0:1787695953.783933:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:2.0:1787695953.783935:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695953.783936:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.783937:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695953.783937:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787695953.783942:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695953.783942:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695953.783944:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.783947:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:2.0:1787695953.783949:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663000. 00000100:00000001:1.0:1787695953.783951:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695953.783953:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.783955:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00100000:2.0:1787695953.783959:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465aed80 x1874531139551616/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1787695953.783965:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ac000 x1874531139551104/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.783987:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695953.783989:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695953.783990:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695953.783993:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695953.783996:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695953.784001:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1787695953.784001:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bed91c58] to pc [ptlrpcd_rcv+-1] req@ffff8b14465aed80 x1874531139551616/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695953.784009:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695953.784014:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695953.784019:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139551104:192.168.201.140@tcp:38:kworker.0 00000100:00080000:2.0:1787695953.784019:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695953.784026:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787695953.784029:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787695953.784029:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695953.784033:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1787695953.784034:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787695953.784036:0:47074:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1787695953.784038:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695953.784040:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3841 02000000:00000001:1.0:1787695953.784044:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000400:00000200:2.0:1787695953.784047:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000010:1.0:1787695953.784051:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 00000400:00000200:2.0:1787695953.784051:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695953.784054:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1787695953.784057:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695953.784059:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695953.784062:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695953.784066:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:2.0:1787695953.784066:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695953.784069:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695953.784070:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695953.784071:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000020:00000001:2.0:1787695953.784072:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695953.784073:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:2.0:1787695953.784078:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000400:00000010:1.0:1787695953.784079:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000020:00000001:2.0:1787695953.784080:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:2.0:1787695953.784085:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695953.784087:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1787695953.784089:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000040:2.0:1787695953.784090:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695953.784095:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000200:1.0:1787695953.784097:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139551104, portal 10 00000100:00000001:1.0:1787695953.784101:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000040:2.0:1787695953.784105:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00000001:1.0:1787695953.784106:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928800256 : -128555780751360 : ffff8b14465ac000) 00000100:00080000:2.0:1787695953.784110:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695953.784114:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695953.784115:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695953.784118:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1787695953.784120:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ac000 x1874531139551104/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695978 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695953.784121:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695953.784124:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695953.784127:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787695953.784133:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695953.784136:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695953.784141:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ae300. 00000100:00000001:1.0:1787695953.784142:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:2.0:1787695953.784145:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000040:1.0:1787695953.784150:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 02000000:00000001:2.0:1787695953.784151:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695953.784155:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695953.784157:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000001:2.0:1787695953.784157:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695953.784159:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695953.784163:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695953.784164:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139551104, offset 0 02000000:00000010:2.0:1787695953.784167:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661400. 00000100:00000001:2.0:1787695953.784171:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695953.784172:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00100000:2.0:1787695953.784179:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ae300 x1874531139551744/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695953.784181:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695953.784189:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1787695953.784191:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:2.0:1787695953.784192:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695953.784200:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1787695953.784201:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004515329a] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ae300 x1874531139551744/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695953.784209:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:2.0:1787695953.784211:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695953.784214:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695953.784215:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000040:2.0:1787695953.784230:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1787695953.784235:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695953.784238:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000040:2.0:1787695953.784239:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1787695953.784244:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17115:17115:256:4294967295] 192.168.201.140@tcp LPNI seq info [17114:17114:8:4294967295] 00000100:00000040:2.0:1787695953.784250:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3851) 00000400:00000200:1.0:1787695953.784253:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695953.784262:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695953.784269:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695953.784273:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695953.784277:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695953.784280:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1787695953.784284:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695953.784287:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1787695953.784291:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695953.784293:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695953.784296:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695953.784300:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695953.784303:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695953.784305:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695953.784314:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695953.784318:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ac000 x1874531139551104/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695978 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.784326:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695953.784329:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ac000 x1874531139551104/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695978 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.784337:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.784338:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695953.784341:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695953.784345:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000400:00000200:1.0:1787695953.784348:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695953.784351:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298515398 00000100:00000001:1.0:1787695953.784353:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298525497 00000100:00000001:1.0:1787695953.784356:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298515398 00000100:00000001:1.0:1787695953.784358:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298525497 00000100:00000001:1.0:1787695953.784360:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695953.784362:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.784395:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695953.784400:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695953/real 1787695953] req@ffff8b14465ac000 x1874531139551104/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695953.784409:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695953.784411:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695953.784413:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695953.784417:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695953.784419:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695953.784421:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695953.784425:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ac000 x1874531139551104/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695953.784433:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ac000 x1874531139551104/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.784451:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695953.784453:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.784455:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695953.784456:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695953.784461:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ac000 x1874531139551104/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.784468:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695953.784472:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ac000 x1874531139551104/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.784480:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695953.784483:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.784484:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695953.784485:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.784488:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695953.784490:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695953.784492:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695953.784496:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695953.784499:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695953.784503:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ac000 x1874531139551104/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695953.784511:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139551104:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695953.784515:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695953.784519:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ac380 x1874531139551232/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.784526:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695953.784528:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695953.784534:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695953.784537:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695953.784539:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695953.784543:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139551232:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695953.784547:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695953.784549:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695953.784550:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695953.784552:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695953.784555:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695953.784557:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695953.784560:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695953.784563:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695953.784566:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695953.784570:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139551232, portal 10 00000100:00000001:1.0:1787695953.784572:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695953.784573:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928801152 : -128555780750464 : ffff8b14465ac380) 00000100:00000040:1.0:1787695953.784580:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ac380 x1874531139551232/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695978 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.784587:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695953.784589:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695953.784592:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695953.784594:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139551232, offset 0 00000400:00000200:1.0:1787695953.784599:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695953.784603:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695953.784607:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695953.784610:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695953.784614:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695953.784616:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695953.784619:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695953.784622:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17116:17116:256:4294967295] 192.168.201.140@tcp LPNI seq info [17115:17115:8:4294967295] 00000400:00000200:1.0:1787695953.784627:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695953.784634:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695953.784645:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695953.784648:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695953.784652:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695953.784655:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1787695953.784659:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695953.784662:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1787695953.784666:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695953.784668:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695953.784672:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695953.784676:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695953.784679:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695953.784681:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695953.784683:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695953.784688:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ac380 x1874531139551232/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695978 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.784696:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695953.784699:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ac380 x1874531139551232/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695978 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.784706:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.784707:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695953.784711:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695953.784714:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787695953.784718:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695953.784720:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298514247 00000100:00000001:1.0:1787695953.784722:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298525497 00000100:00000001:1.0:1787695953.784725:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298514247 00000100:00000001:1.0:1787695953.784727:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298525497 00000100:00000001:1.0:1787695953.784729:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695953.784731:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.784733:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695953.784737:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695953/real 1787695953] req@ffff8b14465ac380 x1874531139551232/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695953.784753:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695953.784755:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695953.784757:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695953.784759:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695953.784761:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695953.784763:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695953.784767:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ac380 x1874531139551232/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695953.784776:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ac380 x1874531139551232/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.784783:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695953.784785:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.784787:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695953.784788:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695953.784792:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ac380 x1874531139551232/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.784799:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695953.784803:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ac380 x1874531139551232/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.784810:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695953.784812:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.784814:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695953.784815:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.784817:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695953.784819:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695953.784822:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695953.784825:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695953.784828:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695953.784839:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ac380 x1874531139551232/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695953.784848:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139551232:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695953.784852:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695953.784855:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465af100 x1874531139551360/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.784861:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695953.784863:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695953.784865:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695953.784867:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695953.784869:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695953.784872:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139551360:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695953.784875:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695953.784877:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695953.784878:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695953.784880:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695953.784883:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695953.784885:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695953.784889:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695953.784892:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695953.784896:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695953.784899:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139551360, portal 4 00000100:00000001:1.0:1787695953.784901:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695953.784903:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928812800 : -128555780738816 : ffff8b14465af100) 00000100:00000040:1.0:1787695953.784909:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465af100 x1874531139551360/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695978 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.784916:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695953.784918:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695953.784921:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695953.784923:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139551360, offset 0 00000400:00000200:1.0:1787695953.784930:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695953.784933:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695953.784936:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695953.784938:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695953.784940:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695953.784941:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695953.784943:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695953.784945:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17117:17117:256:4294967295] 192.168.201.140@tcp LPNI seq info [17116:17116:8:4294967295] 00000400:00000200:1.0:1787695953.784948:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695953.784952:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695953.784955:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695953.784957:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695953.784959:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695953.784962:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1787695953.784964:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695953.784966:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1787695953.784969:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695953.784970:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695953.784972:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695953.784975:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695953.784977:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695953.784978:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695953.784980:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695953.784983:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465af100 x1874531139551360/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695978 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.784988:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695953.784990:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465af100 x1874531139551360/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695978 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.784994:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.784995:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695953.784997:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695953.785000:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695953.785002:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695953.785007:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298514277 00000100:00000001:1.0:1787695953.785009:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298525497 00000100:00000001:1.0:1787695953.785010:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298514277 00000100:00000001:1.0:1787695953.785012:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298525497 00000100:00000001:1.0:1787695953.785013:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695953.785015:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.785016:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695953.785019:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695953/real 1787695953] req@ffff8b14465af100 x1874531139551360/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695953.785024:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695953.785025:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695953.785026:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695953.785027:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695953.785028:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695953.785030:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695953.785032:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465af100 x1874531139551360/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695953.785037:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465af100 x1874531139551360/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785041:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695953.785042:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785043:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695953.785044:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695953.785046:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465af100 x1874531139551360/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785050:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695953.785052:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465af100 x1874531139551360/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785056:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695953.785061:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785062:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695953.785063:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785064:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695953.785065:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695953.785067:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695953.785069:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695953.785070:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695953.785073:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465af100 x1874531139551360/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695953.785078:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139551360:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695953.785080:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695953.785082:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465aea00 x1874531139551488/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785086:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695953.785087:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695953.785088:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695953.785090:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695953.785091:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695953.785092:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139551488:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695953.785094:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695953.785095:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695953.785096:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695953.785098:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695953.785100:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695953.785101:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695953.785103:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695953.785105:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695953.785107:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695953.785109:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139551488, portal 4 00000100:00000001:1.0:1787695953.785113:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695953.785114:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928811008 : -128555780740608 : ffff8b14465aea00) 00000100:00000040:1.0:1787695953.785118:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465aea00 x1874531139551488/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695978 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785122:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695953.785124:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695953.785125:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695953.785127:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139551488, offset 0 00000400:00000200:1.0:1787695953.785129:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695953.785132:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695953.785134:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695953.785136:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695953.785137:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695953.785139:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695953.785140:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695953.785142:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17118:17118:256:4294967295] 192.168.201.140@tcp LPNI seq info [17117:17117:8:4294967295] 00000400:00000200:1.0:1787695953.785145:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695953.785149:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695953.785152:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695953.785154:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695953.785156:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695953.785158:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1787695953.785161:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695953.785165:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1787695953.785168:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695953.785169:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695953.785171:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695953.785174:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695953.785176:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695953.785177:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695953.785179:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695953.785181:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465aea00 x1874531139551488/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695978 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785194:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695953.785196:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465aea00 x1874531139551488/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695978 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785201:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.785202:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695953.785204:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695953.785206:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695953.785208:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695953.785210:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298513949 00000100:00000001:1.0:1787695953.785211:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298525497 00000100:00000001:1.0:1787695953.785213:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298513949 00000100:00000001:1.0:1787695953.785214:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298525497 00000100:00000001:1.0:1787695953.785215:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695953.785217:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.785218:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695953.785235:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695953/real 1787695953] req@ffff8b14465aea00 x1874531139551488/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695953.785243:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695953.785244:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695953.785246:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695953.785248:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695953.785250:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695953.785252:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695953.785256:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465aea00 x1874531139551488/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695953.785261:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465aea00 x1874531139551488/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785266:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695953.785271:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785272:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695953.785273:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695953.785276:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465aea00 x1874531139551488/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785281:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695953.785283:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465aea00 x1874531139551488/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785287:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695953.785289:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785289:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695953.785290:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785291:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695953.785292:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695953.785294:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695953.785296:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695953.785298:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695953.785300:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465aea00 x1874531139551488/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695953.785305:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139551488:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695953.785307:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785309:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695953.785311:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ac000 x1874531139551104/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785314:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695953.785316:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695953.785317:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695953.785319:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695953.785320:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695953.785324:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695953.785325:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695953.785327:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445662000. 02000000:00000001:1.0:1787695953.785329:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695953.785330:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695953.785331:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ac000. 00000100:00000001:1.0:1787695953.785333:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695953.785334:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785336:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695953.785338:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ac380 x1874531139551232/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785342:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695953.785343:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695953.785344:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695953.785345:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695953.785346:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695953.785347:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695953.785348:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695953.785349:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445660400. 02000000:00000001:1.0:1787695953.785350:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695953.785350:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695953.785351:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ac380. 00000100:00000001:1.0:1787695953.785352:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695953.785353:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785354:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695953.785356:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465af100 x1874531139551360/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785360:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695953.785360:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695953.785361:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695953.785380:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695953.785382:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695953.785385:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695953.785388:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695953.785390:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663400. 02000000:00000001:1.0:1787695953.785393:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695953.785397:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695953.785398:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465af100. 00000100:00000001:1.0:1787695953.785401:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695953.785402:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785404:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695953.785407:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465aea00 x1874531139551488/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785411:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695953.785413:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695953.785414:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695953.785416:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695953.785417:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695953.785418:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695953.785419:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695953.785421:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661000. 02000000:00000001:1.0:1787695953.785423:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695953.785424:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695953.785425:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465aea00. 00000100:00000001:1.0:1787695953.785427:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695953.785428:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785429:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785432:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695953.785434:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.785436:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695953.785437:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695953.785438:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695953.785440:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465aed80 x1874531139551616/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785445:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695953.785446:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695953.785448:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695953.785449:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695953.785450:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695953.785452:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139551616:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695953.785454:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695953.785459:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695953.785460:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695953.785461:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695953.785463:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695953.785464:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695953.785467:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695953.785469:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695953.785471:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695953.785473:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139551616, portal 10 00000100:00000001:1.0:1787695953.785475:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695953.785476:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928811904 : -128555780739712 : ffff8b14465aed80) 00000100:00000040:1.0:1787695953.785480:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465aed80 x1874531139551616/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695978 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785484:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695953.785486:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695953.785487:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695953.785489:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139551616, offset 0 00000400:00000200:1.0:1787695953.785491:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695953.785494:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695953.785496:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695953.785498:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695953.785500:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695953.785501:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695953.785503:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695953.785505:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17119:17119:256:4294967295] 192.168.201.140@tcp LPNI seq info [17118:17118:8:4294967295] 00000400:00000200:1.0:1787695953.785508:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695953.785512:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695953.785515:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695953.785517:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695953.785520:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695953.785522:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1787695953.785525:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695953.785526:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1787695953.785529:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695953.785533:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695953.785535:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695953.785538:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695953.785540:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695953.785541:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695953.785543:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695953.785545:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465aed80 x1874531139551616/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695978 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785550:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695953.785552:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465aed80 x1874531139551616/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695978 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785557:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.785558:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695953.785560:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695953.785562:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695953.785564:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695953.785565:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298514136 00000100:00000001:1.0:1787695953.785567:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298525497 00000100:00000001:1.0:1787695953.785569:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298514136 00000100:00000001:1.0:1787695953.785570:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298525497 00000100:00000001:1.0:1787695953.785571:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695953.785573:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.785574:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695953.785576:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695953/real 1787695953] req@ffff8b14465aed80 x1874531139551616/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695953.785581:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695953.785582:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695953.785583:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695953.785584:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695953.785585:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695953.785590:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695953.785592:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465aed80 x1874531139551616/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695953.785598:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465aed80 x1874531139551616/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785601:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695953.785602:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785603:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695953.785604:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695953.785606:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465aed80 x1874531139551616/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785610:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695953.785613:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465aed80 x1874531139551616/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785617:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695953.785619:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785620:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695953.785620:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785622:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695953.785623:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695953.785624:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695953.785626:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695953.785627:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695953.785630:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465aed80 x1874531139551616/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695953.785634:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139551616:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695953.785636:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695953.785642:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ae300 x1874531139551744/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785647:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695953.785648:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695953.785649:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695953.785650:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695953.785651:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695953.785653:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139551744:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695953.785655:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695953.785657:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695953.785658:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695953.785659:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695953.785661:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695953.785662:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695953.785664:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695953.785666:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695953.785668:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695953.785670:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139551744, portal 10 00000100:00000001:1.0:1787695953.785671:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695953.785672:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928809216 : -128555780742400 : ffff8b14465ae300) 00000100:00000040:1.0:1787695953.785676:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ae300 x1874531139551744/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695978 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785681:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695953.785682:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695953.785684:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695953.785685:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139551744, offset 0 00000400:00000200:1.0:1787695953.785688:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695953.785690:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695953.785692:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695953.785694:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695953.785696:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695953.785698:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695953.785702:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695953.785704:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17120:17120:256:4294967295] 192.168.201.140@tcp LPNI seq info [17119:17119:8:4294967295] 00000400:00000200:1.0:1787695953.785707:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695953.785711:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695953.785714:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695953.785716:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695953.785719:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695953.785721:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 48 secs later) 00000800:00000200:1.1:1787695953.785724:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695953.785725:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 48 secs later) 00000800:00000100:1.0:1787695953.785728:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695953.785729:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695953.785731:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695953.785734:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695953.785735:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695953.785737:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695953.785738:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695953.785741:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ae300 x1874531139551744/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695978 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785746:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695953.785749:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ae300 x1874531139551744/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695978 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785756:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.785757:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695953.785760:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695953.785763:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695953.785766:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695953.785769:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298514114 00000100:00000001:1.0:1787695953.785771:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298525497 00000100:00000001:1.0:1787695953.785773:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298514114 00000100:00000001:1.0:1787695953.785776:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298525497 00000100:00000001:1.0:1787695953.785784:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695953.785786:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.785788:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695953.785792:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695953/real 1787695953] req@ffff8b14465ae300 x1874531139551744/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695953.785801:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695953.785802:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695953.785804:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695953.785806:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695953.785808:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695953.785810:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695953.785814:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ae300 x1874531139551744/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695953.785823:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ae300 x1874531139551744/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785829:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695953.785830:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785832:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695953.785833:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695953.785837:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ae300 x1874531139551744/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785844:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695953.785848:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ae300 x1874531139551744/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785855:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695953.785858:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785860:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695953.785861:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785863:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695953.785864:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695953.785871:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695953.785874:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695953.785875:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695953.785878:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ae300 x1874531139551744/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695953.785883:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139551744:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695953.785886:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785887:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695953.785889:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465aed80 x1874531139551616/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785893:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695953.785894:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695953.785896:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695953.785897:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695953.785898:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695953.785899:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695953.785900:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695953.785901:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663000. 02000000:00000001:1.0:1787695953.785903:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695953.785904:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695953.785905:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465aed80. 00000100:00000001:1.0:1787695953.785907:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695953.785908:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785909:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695953.785911:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ae300 x1874531139551744/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695978 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695953.785915:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695953.785915:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695953.785916:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695953.785917:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695953.785918:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695953.785921:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695953.785922:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695953.785924:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661400. 02000000:00000001:1.0:1787695953.785925:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695953.785926:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695953.785927:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ae300. 00000100:00000001:1.0:1787695953.785928:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695953.785929:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785930:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695953.785933:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695953.785934:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695953.785935:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695953.785936:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695953.846477:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695953.846485:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695953.846488:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695953.846495:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695953.846500:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695953.846505:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695953.846508:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695953.846511:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695953.846515:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695954.486477:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695954.486483:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695954.486488:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695954.486489:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695954.486492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695954.486493:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695954.486496:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695954.486496:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695954.486498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695954.486499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695954.486502:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695954.486502:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695954.486610:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695954.486618:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695954.486624:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695954.870475:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695954.870487:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695954.870490:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695954.870498:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695954.870503:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695954.870508:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695954.870512:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695954.870516:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695954.870519:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695955.510455:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695955.510463:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695955.510468:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695955.510469:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695955.510472:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695955.510473:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695955.510476:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695955.510477:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695955.510479:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695955.510480:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695955.510483:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695955.510484:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695955.510526:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695955.510530:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695955.510533:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695955.894499:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695955.894509:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695955.894533:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695955.894540:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695955.894544:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695955.894548:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695955.894550:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695955.894552:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695955.894555:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695956.534511:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695956.534542:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695956.534555:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695956.534556:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695956.534561:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695956.534565:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695956.534567:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695956.534576:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695956.534579:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695956.534581:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695956.534586:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695956.534588:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695956.534590:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695956.534593:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695956.534598:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695956.534600:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695956.534601:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695956.534603:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695956.534605:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695956.534607:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695956.534609:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695956.534610:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695956.534619:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695956.534621:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695956.534626:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695956.534628:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695956.534632:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695956.534634:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695956.534635:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695956.534638:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695956.534639:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695956.534641:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695956.534643:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695956.534644:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695956.534646:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695956.534649:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695956.534718:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695956.534722:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695956.534731:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695956.918475:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695956.918481:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695956.918483:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695956.918487:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695956.918490:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695956.918493:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695956.918495:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695956.918497:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695956.918499:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695957.558461:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695957.558471:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695957.558477:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695957.558478:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695957.558506:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695957.558507:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695957.558510:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695957.558511:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695957.558514:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695957.558515:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695957.558518:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695957.558519:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695957.558582:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695957.558586:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695957.558589:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695957.942509:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695957.942519:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695957.942521:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695957.942527:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695957.942536:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695957.942540:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695957.942543:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695957.942546:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695957.942548:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695958.582505:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695958.582514:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695958.582520:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695958.582521:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695958.582525:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695958.582526:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695958.582530:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695958.582531:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695958.582533:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695958.582534:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695958.582536:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695958.582537:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695958.582638:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695958.582642:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695958.582645:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787695958.902593:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695958.902609:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787695958.902613:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695958.902618:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695958.902624:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695958.902643:0:47074:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3846 00000400:00000200:2.0:1787695958.902657:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695958.902668:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695958.902675:0:47074:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695958.902681:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695958.902686:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695958.902690:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695958.902693:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695958.902698:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695958.902700:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:2.0:1787695958.902703:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695958.902707:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:2.0:1787695958.902710:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:2.0:1787695958.902712:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:2.0:1787695958.902714:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695958.902716:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695958.902717:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695958.902726:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695958.902730:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:2.0:1787695958.902733:0:47074:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695958.902736:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695958.902740:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695958.902745:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695958.902749:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695958.902777:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787695958.902780:0:47074:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695958.902787:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465adc00. 00000020:00000040:2.0:1787695958.902791:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:2.0:1787695958.902795:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695958.902798:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695958.902801:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695958.902804:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695958.902809:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695958.902817:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445660c00. 00000100:00000001:2.0:1787695958.902826:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695958.902837:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465adc00 x1874531139551872/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695958.902847:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695958.902851:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695958.902855:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000114811d4] to pc [ptlrpcd_rcv+-1] req@ffff8b14465adc00 x1874531139551872/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695958.902877:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695958.902880:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695958.902883:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787695958.902884:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695958.902886:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695958.902889:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695958.902892:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3846 00000400:00000200:2.0:1787695958.902895:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695958.902898:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695958.902900:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695958.902902:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695958.902904:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695958.902906:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695958.902908:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695958.902916:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695958.902917:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:2.0:1787695958.902918:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695958.902920:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:2.0:1787695958.902922:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:2.0:1787695958.902924:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000001:1.0:1787695958.902925:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1787695958.902928:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695958.902932:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695958.902933:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695958.902939:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695958.902939:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695958.902944:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695958.902948:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00000040:1.0:1787695958.902952:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465adc00 x1874531139551872/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695958.902954:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695958.902961:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695958.902965:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695958.902973:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695958.902977:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695958.902982:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695958.902983:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695958.902988:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787695958.902989:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787695958.902993:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:2.0:1787695958.902995:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695958.902999:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695958.902999:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695958.903006:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695958.903008:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ad180. 00000100:00100000:1.0:1787695958.903013:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139551872:192.168.201.140@tcp:250:kworker.0 00000020:00000040:2.0:1787695958.903015:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000100:00000001:1.0:1787695958.903022:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695958.903022:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695958.903027:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695958.903029:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787695958.903030:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695958.903033:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695958.903034:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695958.903037:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695958.903039:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695958.903046:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661400. 02000000:00000010:1.0:1787695958.903049:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547c00. 00000100:00000001:2.0:1787695958.903051:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695958.903057:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ad180 x1874531139552000/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787695958.903065:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695958.903068:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1787695958.903077:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695958.903078:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695958.903086:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006ad773a4] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ad180 x1874531139552000/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1787695958.903087:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695958.903097:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695958.903108:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139551872, portal 25 00000100:00000001:1.0:1787695958.903114:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695958.903114:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.903116:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928807424 : -128555780744192 : ffff8b14465adc00) 00000100:00080000:2.0:1787695958.903120:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695958.903127:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000040:1.0:1787695958.903128:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465adc00 x1874531139551872/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695983 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695958.903143:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695958.903147:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695958.903153:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695958.903159:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1787695958.903161:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3846 00000400:00000200:2.0:1787695958.903170:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:1.0:1787695958.903174:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695958.903174:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695958.903179:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695958.903182:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000010:1.0:1787695958.903184:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000001:2.0:1787695958.903185:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695958.903188:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695958.903191:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000200:1.0:1787695958.903194:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139551872, offset 0 00000020:00000001:2.0:1787695958.903194:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695958.903196:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:2.0:1787695958.903200:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695958.903202:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:2.0:1787695958.903203:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:2.0:1787695958.903206:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000400:00000200:1.0:1787695958.903209:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:2.0:1787695958.903211:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695958.903214:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695958.903216:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695958.903218:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695958.903219:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000400:00000200:1.0:1787695958.903223:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:2.0:1787695958.903224:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695958.903233:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695958.903235:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 02000000:00000001:2.0:1787695958.903235:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695958.903240:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695958.903244:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695958.903245:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1787695958.903246:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695958.903250:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787695958.903252:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695958.903255:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:2.0:1787695958.903256:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695958.903261:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465afb80. 00000400:00000200:1.0:1787695958.903265:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000020:00000040:2.0:1787695958.903266:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:2.0:1787695958.903271:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1787695958.903272:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 02000000:00000001:2.0:1787695958.903273:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695958.903277:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695958.903279:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695958.903282:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695958.903283:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17121:17121:256:4294967295] 192.168.201.140@tcp LPNI seq info [17120:17120:8:4294967295] 02000000:00000010:2.0:1787695958.903287:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663000. 00000100:00000001:2.0:1787695958.903291:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695958.903292:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00100000:2.0:1787695958.903299:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465afb80 x1874531139552128/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695958.903306:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695958.903309:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695958.903319:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b9acb62b] to pc [ptlrpcd_rcv+-1] req@ffff8b14465afb80 x1874531139552128/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695958.903320:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695958.903333:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695958.903337:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695958.903342:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:1.0:1787695958.903343:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000100:00080000:2.0:1787695958.903350:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787695958.903353:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695958.903356:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.0:1787695958.903358:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695958.903397:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 43 secs later) 00000100:00000001:2.0:1787695958.903406:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:1.1:1787695958.903406:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695958.903412:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 43 secs later) 00000100:00080000:2.0:1787695958.903414:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3846 00000400:00000200:2.0:1787695958.903428:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000800:00000100:1.0:1787695958.903429:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695958.903434:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:2.0:1787695958.903441:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000200:1.0:1787695958.903445:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:2.0:1787695958.903445:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695958.903452:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695958.903455:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695958.903460:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00000040:2.0:1787695958.903461:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695958.903463:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695958.903465:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695958.903466:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:2.0:1787695958.903468:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695958.903471:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:2.0:1787695958.903473:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:2.0:1787695958.903475:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000400:00000200:1.0:1787695958.903477:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000100:00000040:2.0:1787695958.903478:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695958.903483:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1787695958.903484:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000040:2.0:1787695958.903485:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695958.903489:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695958.903491:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00000001:1.0:1787695958.903494:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1787695958.903494:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695958.903499:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695958.903501:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:1.0:1787695958.903504:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465adc00 x1874531139551872/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695983 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695958.903505:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695958.903510:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695958.903513:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695958.903516:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787695958.903521:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695958.903524:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695958.903528:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ad500. 00000100:00000001:1.0:1787695958.903529:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000040:2.0:1787695958.903530:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:2.0:1787695958.903535:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000040:1.0:1787695958.903536:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465adc00 x1874531139551872/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695983 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695958.903539:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695958.903544:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787695958.903549:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.903552:0:2402:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1787695958.903554:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695958.903558:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.903561:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695958.903565:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661000. 00000400:00000200:1.0:1787695958.903569:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695958.903573:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.903575:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298530617 00000100:00100000:2.0:1787695958.903584:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ad500 x1874531139552256/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695958.903587:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298525497 00000100:00000001:2.0:1787695958.903597:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695958.903598:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298530617 00000100:00000001:2.0:1787695958.903601:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695958.903604:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004ab396cc] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ad500 x1874531139552256/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.903614:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298525497 00000100:00000001:2.0:1787695958.903615:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695958.903622:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695958.903623:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695958.903629:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695958.903630:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1787695958.903635:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787695958.903636:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695958.903640:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787695958.903646:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695958/real 1787695958] req@ffff8b14465adc00 x1874531139551872/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695958.903647:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695958.903654:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3846 00000400:00000200:2.0:1787695958.903664:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000001:1.0:1787695958.903666:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000200:2.0:1787695958.903670:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000001:1.0:1787695958.903671:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:2.0:1787695958.903673:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000400:00000010:1.0:1787695958.903676:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000100:00000040:2.0:1787695958.903679:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695958.903682:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000100:00000001:2.0:1787695958.903682:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695958.903687:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000010:1.0:1787695958.903688:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:2.0:1787695958.903689:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695958.903693:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695958.903694:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000040:2.0:1787695958.903695:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:2.0:1787695958.903698:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695958.903700:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000100:00000200:1.0:1787695958.903702:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465adc00 x1874531139551872/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787695958.903702:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:2.0:1787695958.903707:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000200:1.0:1787695958.903718:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465adc00 x1874531139551872/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695958.903719:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695958.903725:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695958.903730:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695958.903735:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695958.903738:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:2.0:1787695958.903743:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695958.903751:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695958.903755:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695958.903759:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:2.0:1787695958.903761:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695958.903763:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787695958.903768:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.903769:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695958.903773:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695958.903774:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1787695958.903777:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787695958.903782:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695958.903783:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465adc00 x1874531139551872/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695958.903784:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695958.903790:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465aca80. 00000020:00000040:2.0:1787695958.903795:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:2.0:1787695958.903799:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695958.903802:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.903803:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695958.903804:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695958.903808:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695958.903810:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695958.903814:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465adc00 x1874531139551872/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1787695958.903816:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663400. 00000100:00000001:2.0:1787695958.903821:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695958.903830:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465aca80 x1874531139552384/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695958.903831:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695958.903839:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695958.903843:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695958.903844:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695958.903848:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.903849:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695958.903854:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004fad31e4] to pc [ptlrpcd_rcv+-1] req@ffff8b14465aca80 x1874531139552384/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.903855:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695958.903860:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695958.903868:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1787695958.903868:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695958.903873:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695958.903877:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695958.903880:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:2.0:1787695958.903887:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787695958.903890:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:1.0:1787695958.903891:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465adc00 x1874531139551872/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695958.903892:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695958.903898:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695958.903903:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3846 00000400:00000200:2.0:1787695958.903912:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00100000:1.0:1787695958.903913:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139551872:192.168.201.140@tcp:250:kworker.0 00000400:00000200:2.0:1787695958.903916:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695958.903920:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695958.903923:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695958.903924:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695958.903930:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787695958.903930:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695958.903937:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695958.903939:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465adc00 x1874531139551872/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695958.903942:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695958.903948:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695958.903951:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:2.0:1787695958.903956:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695958.903957:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000020:00000040:2.0:1787695958.903960:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 02000000:00000001:1.0:1787695958.903962:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:2.0:1787695958.903965:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000020:00000001:2.0:1787695958.903969:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 02000000:00000010:1.0:1787695958.903973:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547c00. 00000100:00000040:2.0:1787695958.903976:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695958.903979:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787695958.903981:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:1.0:1787695958.903983:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000040:2.0:1787695958.903985:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000020:00000040:1.0:1787695958.903988:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000100:00000001:2.0:1787695958.903990:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695958.903993:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000020:00000040:2.0:1787695958.903996:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 02000000:00000010:1.0:1787695958.903998:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445660c00. 00000100:00080000:2.0:1787695958.904004:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000001:1.0:1787695958.904005:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695958.904009:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787695958.904010:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695958.904013:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465adc00. 02000000:00000001:2.0:1787695958.904015:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695958.904019:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 02000000:00000001:2.0:1787695958.904021:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695958.904022:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787695958.904027:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.904028:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695958.904033:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695958.904036:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787695958.904040:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695958.904045:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787695958.904045:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695958.904050:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.904052:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:2.0:1787695958.904054:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ad880. 00000100:00000001:1.0:1787695958.904056:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000040:2.0:1787695958.904058:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000100:00000001:1.0:1787695958.904060:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787695958.904062:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695958.904064:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695958.904066:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695958.904068:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1787695958.904069:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ad180 x1874531139552000/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695958.904071:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695958.904076:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445660400. 00000100:00000001:2.0:1787695958.904082:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.904088:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695958.904090:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695958.904093:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ad880 x1874531139552512/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787695958.904094:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695958.904097:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695958.904102:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695958.904110:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139552000:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695958.904112:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695958.904117:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787695958.904118:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695958.904121:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695958.904124:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695958.904127:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000075b4d632] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ad880 x1874531139552512/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695958.904128:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695958.904144:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695958.904149:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695958.904158:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695958.904163:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000100:00000001:2.0:1787695958.904168:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695958.904170:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00080000:2.0:1787695958.904173:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1787695958.904176:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139552000, portal 10 00000100:00080000:2.0:1787695958.904179:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787695958.904182:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695958.904185:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695958.904186:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695958.904190:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928804736 : -128555780746880 : ffff8b14465ad180) 00000100:00000001:2.0:1787695958.904191:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695958.904197:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3846 00000100:00000040:1.0:1787695958.904202:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ad180 x1874531139552000/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695983 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787695958.904205:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695958.904213:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695958.904217:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1787695958.904219:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695958.904224:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000040:2.0:1787695958.904225:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000010:1.0:1787695958.904229:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000001:2.0:1787695958.904229:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695958.904234:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000200:1.0:1787695958.904236:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139552000, offset 0 00000100:00000001:2.0:1787695958.904243:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695958.904245:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000020:00000001:2.0:1787695958.904248:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695958.904251:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000020:00000001:2.0:1787695958.904254:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:1.0:1787695958.904255:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000040:2.0:1787695958.904256:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:2.0:1787695958.904261:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000400:00000200:1.0:1787695958.904264:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000020:00000001:2.0:1787695958.904265:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000400:00000200:1.0:1787695958.904273:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1787695958.904273:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695958.904278:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695958.904281:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000040:2.0:1787695958.904282:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695958.904286:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695958.904288:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000020:00000040:2.0:1787695958.904289:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000400:00000200:1.0:1787695958.904297:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00080000:2.0:1787695958.904297:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695958.904302:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695958.904305:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1787695958.904308:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17122:17122:256:4294967295] 192.168.201.140@tcp LPNI seq info [17121:17121:8:4294967295] 02000000:00000001:2.0:1787695958.904311:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1787695958.904317:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:2.0:1787695958.904317:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695958.904323:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695958.904328:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1787695958.904330:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000008:00000001:2.0:1787695958.904332:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695958.904338:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695958.904342:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000010:2.0:1787695958.904346:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465af800. 00000800:00000010:1.0:1787695958.904349:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000020:00000040:2.0:1787695958.904351:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000800:00000200:1.0:1787695958.904357:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:2.0:1787695958.904358:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695958.904362:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695958.904394:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 43 secs later) 00000100:00000001:2.0:1787695958.904396:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695958.904400:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:1.1:1787695958.904404:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:2.0:1787695958.904405:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1787695958.904410:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 43 secs later) 02000000:00000010:2.0:1787695958.904413:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445662000. 00000800:00000100:1.0:1787695958.904419:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695958.904419:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1787695958.904424:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000100:00100000:2.0:1787695958.904429:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465af800 x1874531139552640/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695958.904431:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695958.904440:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695958.904446:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695958.904450:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:2.0:1787695958.904452:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695958.904454:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787695958.904455:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695958.904464:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ad180 x1874531139552000/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695983 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695958.904465:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b410802f] to pc [ptlrpcd_rcv+-1] req@ffff8b14465af800 x1874531139552640/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695958.904494:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695958.904496:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1787695958.904499:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1787695958.904503:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1787695958.904510:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1787695958.904520:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3856) 00000100:00000001:1.0:1787695958.904523:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695958.904531:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ad180 x1874531139552000/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695983 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.904539:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.904542:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695958.904546:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695958.904551:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000400:00000200:1.0:1787695958.904555:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695958.904557:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298520519 00000100:00000001:1.0:1787695958.904560:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298530617 00000100:00000001:1.0:1787695958.904562:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298520519 00000100:00000001:1.0:1787695958.904564:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298530617 00000100:00000001:1.0:1787695958.904567:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695958.904569:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.904571:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695958.904574:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695958/real 1787695958] req@ffff8b14465ad180 x1874531139552000/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695958.904582:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695958.904583:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695958.904585:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695958.904587:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695958.904589:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695958.904592:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695958.904595:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ad180 x1874531139552000/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695958.904603:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ad180 x1874531139552000/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.904610:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695958.904616:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.904618:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695958.904620:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695958.904623:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ad180 x1874531139552000/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.904631:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695958.904635:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ad180 x1874531139552000/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.904642:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695958.904645:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.904647:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695958.904648:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.904650:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695958.904652:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695958.904655:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695958.904658:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695958.904660:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695958.904664:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ad180 x1874531139552000/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695958.904672:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139552000:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695958.904676:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695958.904679:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465afb80 x1874531139552128/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.904686:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695958.904687:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695958.904689:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695958.904691:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695958.904693:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695958.904703:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139552128:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695958.904707:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695958.904708:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695958.904710:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695958.904712:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695958.904716:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695958.904718:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695958.904721:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695958.904724:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695958.904727:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695958.904731:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139552128, portal 10 00000100:00000001:1.0:1787695958.904733:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695958.904735:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928815488 : -128555780736128 : ffff8b14465afb80) 00000100:00000040:1.0:1787695958.904741:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465afb80 x1874531139552128/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695983 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.904748:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695958.904750:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695958.904753:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695958.904755:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139552128, offset 0 00000400:00000200:1.0:1787695958.904759:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695958.904763:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695958.904767:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695958.904770:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695958.904773:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695958.904775:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695958.904778:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695958.904781:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17123:17123:256:4294967295] 192.168.201.140@tcp LPNI seq info [17122:17122:8:4294967295] 00000400:00000200:1.0:1787695958.904785:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695958.904791:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695958.904796:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695958.904799:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695958.904807:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695958.904810:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1787695958.904815:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695958.904818:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1787695958.904823:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695958.904826:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695958.904830:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695958.904834:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695958.904837:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695958.904840:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695958.904842:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695958.904846:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465afb80 x1874531139552128/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695983 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.904854:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695958.904857:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465afb80 x1874531139552128/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695983 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.904864:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.904866:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695958.904869:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695958.904872:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787695958.904875:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695958.904877:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298519367 00000100:00000001:1.0:1787695958.904879:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298530617 00000100:00000001:1.0:1787695958.904882:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298519367 00000100:00000001:1.0:1787695958.904884:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298530617 00000100:00000001:1.0:1787695958.904886:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695958.904888:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.904890:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695958.904893:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695958/real 1787695958] req@ffff8b14465afb80 x1874531139552128/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695958.904900:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695958.904906:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695958.904908:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695958.904911:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695958.904914:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695958.904917:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695958.904922:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465afb80 x1874531139552128/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695958.904931:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465afb80 x1874531139552128/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.904937:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695958.904939:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.904940:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695958.904942:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695958.904945:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465afb80 x1874531139552128/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.904952:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695958.904955:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465afb80 x1874531139552128/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.904962:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695958.904964:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.904966:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695958.904967:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.904968:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695958.904970:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695958.904973:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695958.904976:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695958.904979:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695958.904983:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465afb80 x1874531139552128/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695958.905000:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139552128:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695958.905005:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695958.905008:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ad500 x1874531139552256/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.905015:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695958.905016:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695958.905019:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695958.905020:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695958.905022:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695958.905025:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139552256:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695958.905029:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695958.905031:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695958.905032:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695958.905034:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695958.905037:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695958.905039:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695958.905043:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695958.905046:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695958.905049:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695958.905052:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139552256, portal 4 00000100:00000001:1.0:1787695958.905054:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695958.905056:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928805632 : -128555780745984 : ffff8b14465ad500) 00000100:00000040:1.0:1787695958.905062:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ad500 x1874531139552256/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695983 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.905069:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695958.905071:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695958.905073:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695958.905077:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139552256, offset 0 00000400:00000200:1.0:1787695958.905081:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695958.905090:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695958.905093:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695958.905096:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695958.905100:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695958.905102:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695958.905105:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695958.905108:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17124:17124:256:4294967295] 192.168.201.140@tcp LPNI seq info [17123:17123:8:4294967295] 00000400:00000200:1.0:1787695958.905113:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695958.905119:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695958.905123:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695958.905126:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695958.905141:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695958.905145:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1787695958.905150:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695958.905153:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1787695958.905156:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695958.905158:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695958.905162:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695958.905165:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695958.905168:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695958.905170:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695958.905172:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695958.905176:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ad500 x1874531139552256/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695983 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.905184:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695958.905187:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ad500 x1874531139552256/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695983 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.905194:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.905196:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695958.905199:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695958.905203:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695958.905206:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695958.905213:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298519397 00000100:00000001:1.0:1787695958.905216:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298530617 00000100:00000001:1.0:1787695958.905218:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298519397 00000100:00000001:1.0:1787695958.905220:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298530617 00000100:00000001:1.0:1787695958.905222:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695958.905225:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.905227:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695958.905230:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695958/real 1787695958] req@ffff8b14465ad500 x1874531139552256/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695958.905238:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695958.905240:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695958.905242:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695958.905244:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695958.905246:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695958.905248:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695958.905251:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ad500 x1874531139552256/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695958.905260:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ad500 x1874531139552256/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.905267:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695958.905268:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.905270:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695958.905271:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695958.905274:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ad500 x1874531139552256/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.905280:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695958.905284:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ad500 x1874531139552256/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.905291:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695958.905298:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.905299:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695958.905301:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.905302:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695958.905304:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695958.905307:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695958.905310:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695958.905313:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695958.905317:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ad500 x1874531139552256/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695958.905325:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139552256:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695958.905329:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695958.905332:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465aca80 x1874531139552384/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.905361:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695958.905385:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695958.905389:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695958.905391:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695958.905394:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695958.905397:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139552384:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695958.905400:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695958.905402:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695958.905404:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695958.905406:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695958.905410:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695958.905413:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695958.905417:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695958.905420:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695958.905424:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695958.905427:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139552384, portal 4 00000100:00000001:1.0:1787695958.905436:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695958.905437:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928802944 : -128555780748672 : ffff8b14465aca80) 00000100:00000040:1.0:1787695958.905444:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465aca80 x1874531139552384/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695983 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.905451:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695958.905453:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695958.905456:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695958.905458:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139552384, offset 0 00000400:00000200:1.0:1787695958.905462:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695958.905467:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695958.905471:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695958.905474:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695958.905476:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695958.905479:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695958.905481:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695958.905485:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17125:17125:256:4294967295] 192.168.201.140@tcp LPNI seq info [17124:17124:8:4294967295] 00000400:00000200:1.0:1787695958.905489:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695958.905496:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695958.905500:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695958.905504:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695958.905508:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695958.905511:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1787695958.905515:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695958.905518:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1787695958.905522:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695958.905523:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695958.905526:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695958.905530:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695958.905533:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695958.905535:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695958.905538:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695958.905541:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465aca80 x1874531139552384/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695983 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.905559:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695958.905563:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465aca80 x1874531139552384/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695983 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.905569:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.905571:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695958.905574:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695958.905577:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695958.905580:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695958.905582:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298519070 00000100:00000001:1.0:1787695958.905585:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298530617 00000100:00000001:1.0:1787695958.905588:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298519070 00000100:00000001:1.0:1787695958.905590:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298530617 00000100:00000001:1.0:1787695958.905592:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695958.905594:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.905597:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695958.905600:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695958/real 1787695958] req@ffff8b14465aca80 x1874531139552384/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695958.905608:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695958.905609:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695958.905611:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695958.905614:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695958.905615:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695958.905618:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695958.905621:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465aca80 x1874531139552384/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695958.905630:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465aca80 x1874531139552384/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.905636:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695958.905641:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.905644:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695958.905645:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695958.905649:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465aca80 x1874531139552384/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.905656:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695958.905660:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465aca80 x1874531139552384/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.905667:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695958.905670:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.905671:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695958.905673:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.905675:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695958.905677:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695958.905679:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695958.905682:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695958.905685:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695958.905689:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465aca80 x1874531139552384/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695958.905696:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139552384:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695958.905700:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.905702:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695958.905705:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ad180 x1874531139552000/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.905712:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695958.905714:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695958.905716:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695958.905719:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695958.905720:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695958.905725:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695958.905728:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695958.905730:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661400. 02000000:00000001:1.0:1787695958.905733:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695958.905735:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695958.905737:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ad180. 00000100:00000001:1.0:1787695958.905739:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695958.905741:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.905743:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695958.905747:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465afb80 x1874531139552128/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.905753:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695958.905755:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695958.905756:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695958.905758:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695958.905759:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695958.905761:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695958.905763:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695958.905765:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663000. 02000000:00000001:1.0:1787695958.905767:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695958.905769:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695958.905770:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465afb80. 00000100:00000001:1.0:1787695958.905773:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695958.905774:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.905775:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695958.905778:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ad500 x1874531139552256/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.905786:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695958.905787:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695958.905789:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695958.905793:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695958.905794:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695958.905796:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695958.905797:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695958.905798:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661000. 02000000:00000001:1.0:1787695958.905800:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695958.905805:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695958.905807:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ad500. 00000100:00000001:1.0:1787695958.905809:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695958.905811:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.905812:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695958.905816:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465aca80 x1874531139552384/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.905822:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695958.905824:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695958.905825:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695958.905828:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695958.905829:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695958.905830:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695958.905832:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695958.905833:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663400. 02000000:00000001:1.0:1787695958.905835:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695958.905836:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695958.905838:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465aca80. 00000100:00000001:1.0:1787695958.905841:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695958.905842:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.905844:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.905848:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695958.905849:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.905852:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695958.905853:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695958.905854:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695958.905858:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ad880 x1874531139552512/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.905865:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695958.905866:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695958.905869:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695958.905871:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695958.905873:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695958.905876:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139552512:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695958.905879:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695958.905884:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695958.905886:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695958.905888:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695958.905890:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695958.905892:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695958.905896:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695958.905899:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695958.905902:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695958.905905:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139552512, portal 10 00000100:00000001:1.0:1787695958.905907:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695958.905909:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928806528 : -128555780745088 : ffff8b14465ad880) 00000100:00000040:1.0:1787695958.905915:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ad880 x1874531139552512/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695983 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.905922:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695958.905924:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695958.905927:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695958.905929:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139552512, offset 0 00000400:00000200:1.0:1787695958.905933:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695958.905937:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695958.905940:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695958.905943:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695958.905946:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695958.905948:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695958.905951:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695958.905954:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17126:17126:256:4294967295] 192.168.201.140@tcp LPNI seq info [17125:17125:8:4294967295] 00000400:00000200:1.0:1787695958.905959:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695958.905966:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695958.905971:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695958.905974:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695958.905978:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695958.905980:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1787695958.905984:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695958.905987:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1787695958.905991:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695958.905997:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695958.906001:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695958.906004:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695958.906007:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695958.906009:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695958.906012:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695958.906015:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ad880 x1874531139552512/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695983 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.906023:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695958.906027:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ad880 x1874531139552512/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695983 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.906033:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.906035:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695958.906038:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695958.906041:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695958.906044:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695958.906047:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298519256 00000100:00000001:1.0:1787695958.906050:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298530617 00000100:00000001:1.0:1787695958.906052:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298519256 00000100:00000001:1.0:1787695958.906054:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298530617 00000100:00000001:1.0:1787695958.906057:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695958.906058:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.906060:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695958.906065:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695958/real 1787695958] req@ffff8b14465ad880 x1874531139552512/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695958.906073:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695958.906074:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695958.906076:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695958.906078:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695958.906080:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695958.906090:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695958.906094:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ad880 x1874531139552512/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695958.906102:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ad880 x1874531139552512/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.906109:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695958.906111:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.906113:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695958.906114:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695958.906117:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ad880 x1874531139552512/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.906124:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695958.906128:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ad880 x1874531139552512/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.906147:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695958.906149:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.906151:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695958.906152:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.906154:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695958.906156:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695958.906158:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695958.906162:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695958.906164:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695958.906168:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ad880 x1874531139552512/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695958.906177:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139552512:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695958.906181:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695958.906189:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465af800 x1874531139552640/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.906196:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695958.906198:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695958.906200:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695958.906202:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695958.906204:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695958.906207:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139552640:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695958.906210:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695958.906212:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695958.906214:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695958.906216:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695958.906218:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695958.906220:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695958.906224:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695958.906226:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695958.906230:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695958.906233:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139552640, portal 10 00000100:00000001:1.0:1787695958.906235:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695958.906237:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928814592 : -128555780737024 : ffff8b14465af800) 00000100:00000040:1.0:1787695958.906242:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465af800 x1874531139552640/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695983 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.906249:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695958.906251:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695958.906254:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695958.906256:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139552640, offset 0 00000400:00000200:1.0:1787695958.906260:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695958.906264:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695958.906267:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695958.906270:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695958.906274:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695958.906276:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695958.906283:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695958.906286:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17127:17127:256:4294967295] 192.168.201.140@tcp LPNI seq info [17126:17126:8:4294967295] 00000400:00000200:1.0:1787695958.906290:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695958.906297:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695958.906302:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695958.906304:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695958.906308:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695958.906312:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 43 secs later) 00000800:00000200:1.1:1787695958.906316:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695958.906319:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 43 secs later) 00000800:00000100:1.0:1787695958.906323:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695958.906325:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695958.906328:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695958.906331:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695958.906334:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695958.906336:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695958.906339:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695958.906342:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465af800 x1874531139552640/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695983 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.906350:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695958.906353:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465af800 x1874531139552640/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695983 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.906360:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.906362:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695958.906387:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695958.906391:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695958.906396:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695958.906398:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298519235 00000100:00000001:1.0:1787695958.906401:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298530617 00000100:00000001:1.0:1787695958.906403:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298519235 00000100:00000001:1.0:1787695958.906406:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298530617 00000100:00000001:1.0:1787695958.906413:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695958.906415:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.906417:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695958.906421:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695958/real 1787695958] req@ffff8b14465af800 x1874531139552640/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695958.906430:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695958.906432:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695958.906434:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695958.906436:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695958.906438:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695958.906440:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695958.906443:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465af800 x1874531139552640/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695958.906452:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465af800 x1874531139552640/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.906458:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695958.906460:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.906462:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695958.906463:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695958.906467:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465af800 x1874531139552640/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.906474:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695958.906479:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465af800 x1874531139552640/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.906486:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695958.906489:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.906490:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695958.906492:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.906494:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695958.906495:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695958.906502:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695958.906506:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695958.906508:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695958.906513:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465af800 x1874531139552640/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695958.906520:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139552640:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695958.906524:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.906526:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695958.906529:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ad880 x1874531139552512/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.906536:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695958.906537:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695958.906539:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695958.906542:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695958.906543:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695958.906545:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695958.906547:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695958.906549:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445660400. 02000000:00000001:1.0:1787695958.906551:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695958.906552:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695958.906553:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ad880. 00000100:00000001:1.0:1787695958.906556:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695958.906557:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.906559:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695958.906562:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465af800 x1874531139552640/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695983 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695958.906569:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695958.906570:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695958.906572:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695958.906574:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695958.906575:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695958.906580:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695958.906582:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695958.906584:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445662000. 02000000:00000001:1.0:1787695958.906587:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695958.906588:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695958.906589:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465af800. 00000100:00000001:1.0:1787695958.906592:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695958.906593:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.906595:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695958.906599:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695958.906601:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695958.906603:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695958.906605:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695958.966472:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695958.966478:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695958.966479:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695958.966484:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695958.966487:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695958.966490:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695958.966492:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695958.966494:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695958.966496:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695959.606483:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695959.606493:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695959.606502:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695959.606503:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695959.606508:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695959.606510:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695959.606514:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695959.606516:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695959.606520:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695959.606522:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695959.606526:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695959.606527:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695959.606630:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695959.606636:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695959.606639:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695959.990538:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695959.990573:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695959.990576:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695959.990590:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695959.990594:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695959.990600:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695959.990603:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695959.990606:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695959.990610:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695960.630513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695960.630522:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695960.630529:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695960.630530:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695960.630534:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695960.630535:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695960.630538:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695960.630539:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695960.630542:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695960.630543:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695960.630545:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695960.630546:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695960.630624:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695960.630630:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695960.630635:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695961.014503:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695961.014512:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695961.014514:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695961.014520:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695961.014524:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695961.014528:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695961.014531:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695961.014543:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695961.014547:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695961.654521:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695961.654553:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695961.654561:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695961.654563:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695961.654568:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695961.654570:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695961.654574:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695961.654576:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695961.654581:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695961.654585:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695961.654589:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695961.654597:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695961.654599:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695961.654601:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695961.654606:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695961.654608:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695961.654610:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695961.654612:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695961.654615:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695961.654616:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695961.654735:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695961.654744:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695961.654748:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695962.038477:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695962.038483:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695962.038485:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695962.038490:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695962.038493:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695962.038496:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695962.038498:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695962.038500:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695962.038502:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695962.678469:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695962.678476:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695962.678481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695962.678482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695962.678486:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695962.678487:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695962.678491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695962.678491:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695962.678494:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695962.678495:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695962.678498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695962.678499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695962.678585:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695962.678591:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695962.678594:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695963.062459:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695963.062466:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695963.062467:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695963.062479:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695963.062482:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695963.062486:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695963.062488:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695963.062491:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695963.062494:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695963.702447:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695963.702456:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695963.702459:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695963.702462:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695963.702468:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695963.702470:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695963.702472:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695963.702474:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695963.702499:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695963.702502:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695963.702503:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695963.702505:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695963.702507:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695963.702509:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695963.702511:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695963.702513:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695963.702528:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695963.702531:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695963.702536:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695963.702537:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695963.702542:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695963.702543:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695963.702546:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695963.702547:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695963.702644:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695963.702651:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695963.702655:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787695964.022728:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695964.022744:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787695964.022748:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695964.022753:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695964.022758:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695964.022768:0:47074:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3851 00000400:00000200:3.0:1787695964.022781:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695964.022790:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695964.022795:0:47074:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695964.022798:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695964.022803:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695964.022806:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695964.022809:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695964.022824:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695964.022830:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:3.0:1787695964.022834:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695964.022839:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787695964.022841:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787695964.022843:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787695964.022847:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695964.022849:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695964.022852:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695964.022854:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695964.022857:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787695964.022861:0:47074:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695964.022864:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695964.022868:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695964.022873:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695964.022877:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695964.022880:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787695964.022883:0:47074:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695964.022927:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7100. 00000020:00000040:3.0:1787695964.022932:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787695964.022937:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695964.022940:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695964.022945:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695964.022949:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695964.022957:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695964.022966:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674800. 00000100:00000001:3.0:1787695964.022974:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695964.022985:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7100 x1874531139552768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695964.022998:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695964.023004:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695964.023010:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000004e8abd7] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7100 x1874531139552768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695964.023040:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695964.023057:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695964.023061:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695964.023062:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695964.023064:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695964.023067:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695964.023071:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3851 00000400:00000200:3.0:1787695964.023075:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695964.023079:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695964.023082:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695964.023086:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695964.023089:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695964.023091:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695964.023093:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695964.023096:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695964.023097:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000100:00000001:1.0:1787695964.023099:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695964.023107:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695964.023111:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1787695964.023115:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695964.023119:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:3.0:1787695964.023123:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:1.0:1787695964.023125:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7100 x1874531139552768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695964.023126:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:3.0:1787695964.023134:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695964.023140:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695964.023145:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695964.023149:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695964.023149:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695964.023153:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00000001:1.0:1787695964.023154:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695964.023158:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000001:1.0:1787695964.023163:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787695964.023163:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695964.023169:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695964.023182:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695964.023189:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695964.023192:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695964.023194:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695964.023199:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695964.023202:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695964.023205:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695964.023209:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4e00. 00000020:00000040:3.0:1787695964.023212:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695964.023214:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695964.023215:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695964.023217:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695964.023219:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695964.023221:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695964.023228:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 00000100:00000001:3.0:1787695964.023232:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695964.023238:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4e00 x1874531139552896/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787695964.023246:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695964.023249:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695964.023253:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695964.023257:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139552768:192.168.201.140@tcp:250:kworker.0 00000100:00000040:3.0:1787695964.023262:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcd77cd6] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4e00 x1874531139552896/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.023266:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695964.023272:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695964.023278:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695964.023282:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695964.023283:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1787695964.023289:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000010:1.0:1787695964.023295:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 00000100:00080000:3.0:1787695964.023295:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695964.023300:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1787695964.023301:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695964.023303:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695964.023309:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695964.023313:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3851 00000400:00000200:3.0:1787695964.023318:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695964.023322:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695964.023326:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1787695964.023327:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695964.023333:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000010:1.0:1787695964.023337:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000100:00000001:3.0:1787695964.023337:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695964.023343:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695964.023346:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000010:1.0:1787695964.023349:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000020:00000001:3.0:1787695964.023350:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695964.023352:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:3.0:1787695964.023353:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695964.023356:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:3.0:1787695964.023362:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:3.0:1787695964.023399:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:3.0:1787695964.023403:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695964.023406:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695964.023408:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695964.023411:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695964.023414:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:3.0:1787695964.023418:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695964.023421:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695964.023429:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:1.0:1787695964.023433:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139552768, portal 25 02000000:00000001:3.0:1787695964.023434:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695964.023441:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1787695964.023441:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.023446:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930156288 : -128555779395328 : ffff8b14466f7100) 00000100:00000001:3.0:1787695964.023447:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695964.023451:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695964.023456:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695964.023459:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7100 x1874531139552768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695964.023459:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695964.023466:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7480. 00000020:00000040:3.0:1787695964.023472:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000100:00000001:1.0:1787695964.023479:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787695964.023480:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695964.023483:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695964.023487:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000040:1.0:1787695964.023488:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695964.023490:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1787695964.023495:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 02000000:00000001:3.0:1787695964.023496:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695964.023503:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676000. 00000100:00000200:1.0:1787695964.023504:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139552768, offset 0 00000100:00000001:3.0:1787695964.023510:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695964.023517:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695964.023529:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695964.023536:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695964.023545:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695964.023550:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695964.023554:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695964.023558:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695964.023563:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17128:17128:256:4294967295] 192.168.201.140@tcp LPNI seq info [17127:17127:8:4294967295] 00000400:00000200:1.0:1787695964.023567:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695964.023582:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695964.023591:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695964.023598:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695964.023608:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695964.023621:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1787695964.023628:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695964.023632:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1787695964.023637:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695964.023640:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695964.023649:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695964.023654:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695964.023661:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695964.023664:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695964.023671:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695964.023675:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7100 x1874531139552768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.023695:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695964.023700:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7100 x1874531139552768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695989 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695964.023704:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7480 x1874531139553024/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695964.023718:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.023724:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787695964.023725:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695964.023730:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.023733:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695964.023736:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043daeda0] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7480 x1874531139553024/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695964.023744:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695964.023752:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298535737 00000100:00000001:3.0:1787695964.023755:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695964.023760:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695964.023764:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298530617 00000100:00080000:3.0:1787695964.023765:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787695964.023770:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695964.023773:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695964.023774:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298535737 00000100:00000001:3.0:1787695964.023779:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695964.023782:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298530617 00000100:00080000:3.0:1787695964.023785:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3851 00000100:00000001:1.0:1787695964.023788:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695964.023794:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1787695964.023796:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695964.023799:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000400:00000200:3.0:1787695964.023804:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695964.023809:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787695964.023810:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695964/real 1787695964] req@ffff8b14466f7100 x1874531139552768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695964.023827:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:3.0:1787695964.023829:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000001:1.0:1787695964.023831:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787695964.023835:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1787695964.023837:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000100:00000040:3.0:1787695964.023840:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695964.023844:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000100:00000001:3.0:1787695964.023845:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000010:1.0:1787695964.023849:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000020:00000001:3.0:1787695964.023851:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695964.023855:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000100:00000001:1.0:1787695964.023857:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:3.0:1787695964.023858:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695964.023862:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000100:00000200:1.0:1787695964.023866:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7100 x1874531139552768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695964.023867:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:3.0:1787695964.023874:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:3.0:1787695964.023883:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695964.023888:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1787695964.023890:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7100 x1874531139552768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695964.023893:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695964.023899:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695964.023905:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:3.0:1787695964.023911:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695964.023913:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787695964.023916:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.023918:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787695964.023920:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695964.023924:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1787695964.023927:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695964.023930:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.023936:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695964.023936:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695964.023940:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787695964.023945:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695964.023947:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7100 x1874531139552768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695964.023949:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695964.023957:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6a00. 00000100:00000001:1.0:1787695964.023960:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1787695964.023962:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:3.0:1787695964.023968:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000040:1.0:1787695964.023970:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7100 x1874531139552768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695964.023971:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695964.023975:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695964.023978:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695964.023982:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.023990:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000010:3.0:1787695964.023990:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 00000100:00000001:1.0:1787695964.024001:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695964.024003:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.024006:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695964.024009:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787695964.024013:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6a00 x1874531139553152/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695964.024015:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695964.024020:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695964.024027:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787695964.024031:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00080000:1.0:1787695964.024036:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:3.0:1787695964.024036:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.024057:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1787695964.024059:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002ad47e8f] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6a00 x1874531139553152/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695964.024071:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7100 x1874531139552768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695964.024072:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695964.024079:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695964.024088:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1787695964.024092:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1787695964.024095:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139552768:192.168.201.140@tcp:250:kworker.0 00000100:00080000:3.0:1787695964.024096:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695964.024102:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695964.024108:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695964.024109:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3851 00000100:00000001:1.0:1787695964.024115:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1787695964.024118:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:1.0:1787695964.024125:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7100 x1874531139552768/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695964.024127:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695964.024133:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695964.024141:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695964.024144:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1787695964.024146:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787695964.024149:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:3.0:1787695964.024150:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695964.024154:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000010:1.0:1787695964.024155:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 00000020:00000001:3.0:1787695964.024158:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695964.024162:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 02000000:00000001:1.0:1787695964.024164:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787695964.024165:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787695964.024170:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:3.0:1787695964.024170:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:3.0:1787695964.024174:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:3.0:1787695964.024176:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:3.0:1787695964.024180:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695964.024182:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000040:1.0:1787695964.024186:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:1.0:1787695964.024194:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000040:3.0:1787695964.024195:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695964.024198:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695964.024201:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:3.0:1787695964.024205:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000010:1.0:1787695964.024208:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674800. 00000100:00000001:3.0:1787695964.024209:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695964.024215:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695964.024217:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787695964.024219:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695964.024221:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695964.024225:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7100. 02000000:00000001:3.0:1787695964.024225:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.024230:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787695964.024230:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695964.024233:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1787695964.024235:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787695964.024236:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1787695964.024241:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695964.024244:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.024248:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:3.0:1787695964.024250:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5c00. 00000100:00000001:1.0:1787695964.024253:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695964.024254:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 00000100:00000001:1.0:1787695964.024262:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787695964.024262:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787695964.024267:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787695964.024267:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695964.024270:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787695964.024271:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787695964.024274:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695964.024277:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695964.024282:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4e00 x1874531139552896/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1787695964.024284:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 00000100:00000001:3.0:1787695964.024292:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.024298:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695964.024302:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695964.024303:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5c00 x1874531139553280/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787695964.024308:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695964.024313:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695964.024318:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787695964.024319:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695964.024324:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695964.024329:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139552896:192.168.201.140@tcp:38:kworker.0 00000100:00000040:3.0:1787695964.024333:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005bfbcee1] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5c00 x1874531139553280/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.024337:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695964.024341:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695964.024345:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695964.024350:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787695964.024352:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695964.024356:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 00000100:00080000:3.0:1787695964.024357:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1787695964.024388:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695964.024395:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695964.024399:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695964.024404:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00080000:3.0:1787695964.024405:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695964.024410:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:1.0:1787695964.024412:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139552896, portal 10 00000100:00000001:1.0:1787695964.024417:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787695964.024420:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695964.024421:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930147328 : -128555779404288 : ffff8b14466f4e00) 00000100:00000001:3.0:1787695964.024427:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:1.0:1787695964.024433:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4e00 x1874531139552896/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695964.024434:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3851 00000400:00000200:3.0:1787695964.024445:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695964.024450:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695964.024455:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695964.024461:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695964.024466:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787695964.024466:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695964.024471:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695964.024475:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695964.024476:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695964.024480:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000010:1.0:1787695964.024481:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000020:00000040:3.0:1787695964.024482:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:3.0:1787695964.024486:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000200:1.0:1787695964.024488:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139552896, offset 0 00000020:00000040:3.0:1787695964.024489:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000020:00000001:3.0:1787695964.024494:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000020:00000001:3.0:1787695964.024497:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000200:1.0:1787695964.024498:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695964.024505:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695964.024508:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695964.024509:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695964.024514:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695964.024517:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:3.0:1787695964.024520:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695964.024522:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:3.0:1787695964.024525:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000400:00000200:1.0:1787695964.024527:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695964.024534:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00080000:3.0:1787695964.024535:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:1.0:1787695964.024537:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:3.0:1787695964.024542:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695964.024547:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17129:17129:256:4294967295] 192.168.201.140@tcp LPNI seq info [17128:17128:8:4294967295] 02000000:00000001:3.0:1787695964.024548:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695964.024553:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1787695964.024559:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787695964.024560:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695964.024565:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695964.024568:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695964.024572:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695964.024576:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695964.024578:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695964.024584:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695964.024593:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000100:00000010:3.0:1787695964.024593:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7800. 00000020:00000040:3.0:1787695964.024600:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000800:00000200:1.0:1787695964.024606:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:3.0:1787695964.024607:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695964.024611:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695964.024613:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 38 secs later) 00000100:00000001:3.0:1787695964.024615:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695964.024618:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695964.024622:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1787695964.024623:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695964.024629:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 38 secs later) 02000000:00000010:3.0:1787695964.024630:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675800. 00000800:00000100:1.0:1787695964.024636:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695964.024637:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1787695964.024641:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000100:00100000:3.0:1787695964.024645:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7800 x1874531139553408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695964.024648:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695964.024658:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1787695964.024659:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1787695964.024662:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000100:00000001:3.0:1787695964.024664:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695964.024666:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695964.024671:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1787695964.024674:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7c57981] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7800 x1874531139553408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695964.024676:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4e00 x1874531139552896/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695964.024707:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.024708:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695964.024713:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4e00 x1874531139552896/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695989 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695964.024714:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695964.024722:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695964.024725:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695964.024727:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695964.024729:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695964.024732:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3851 00000400:00000200:3.0:1787695964.024740:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695964.024746:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695964.024750:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695964.024754:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695964.024754:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695964.024758:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695964.024761:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1787695964.024762:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695964.024766:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695964.024769:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695964.024771:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695964.024778:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000020:00000040:3.0:1787695964.024778:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000020:00000001:3.0:1787695964.024782:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695964.024786:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000400:00000200:1.0:1787695964.024788:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:3.0:1787695964.024791:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:1.0:1787695964.024793:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298525639 00000020:00000001:3.0:1787695964.024798:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:1.0:1787695964.024799:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298535737 00000100:00000001:1.0:1787695964.024805:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298525639 00000100:00000001:1.0:1787695964.024808:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298535737 00000100:00000040:3.0:1787695964.024809:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695964.024812:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695964.024816:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695964.024816:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695964.024820:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000040:3.0:1787695964.024823:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00100000:1.0:1787695964.024827:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695964/real 1787695964] req@ffff8b14466f4e00 x1874531139552896/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695964.024829:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695964.024835:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000400:00000001:1.0:1787695964.024843:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:3.0:1787695964.024843:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000001:1.0:1787695964.024846:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695964.024850:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000100:00000001:3.0:1787695964.024850:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695964.024855:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 02000000:00000001:3.0:1787695964.024856:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:1.0:1787695964.024858:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695964.024861:0:2402:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1787695964.024862:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1787695964.024864:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4e00 x1874531139552896/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695964.024868:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695964.024873:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000200:1.0:1787695964.024878:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4e00 x1874531139552896/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787695964.024879:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695964.024886:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.024892:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787695964.024893:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.024896:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.024899:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695964.024900:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:3.0:1787695964.024900:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7b80. 00000020:00000040:3.0:1787695964.024907:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00100000:1.0:1787695964.024911:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4e00 x1874531139552896/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695964.024913:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695964.024915:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695964.024918:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695964.024921:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695964.024924:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695964.024932:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 00000100:00000001:1.0:1787695964.024934:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695964.024938:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695964.024943:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4e00 x1874531139552896/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695964.024948:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7b80 x1874531139553536/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695964.024978:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787695964.024978:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695964.024983:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695964.024983:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.024986:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695964.024987:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.024988:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695964.024991:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:3.0:1787695964.024996:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b87e1da] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7b80 x1874531139553536/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695964.024997:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695964.025005:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695964.025010:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787695964.025014:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695964.025018:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4e00 x1874531139552896/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695964.025019:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787695964.025027:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695964.025032:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787695964.025037:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139552896:192.168.201.140@tcp:38:kworker.0 00000100:00000040:3.0:1787695964.025038:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1787695964.025058:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1787695964.025063:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3861) 00000100:00000040:1.0:1787695964.025065:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7480 x1874531139553024/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.025079:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695964.025081:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695964.025084:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695964.025087:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695964.025090:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695964.025094:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139553024:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695964.025097:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695964.025100:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695964.025108:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695964.025111:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695964.025115:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695964.025118:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695964.025123:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695964.025126:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695964.025131:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695964.025135:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139553024, portal 10 00000100:00000001:1.0:1787695964.025137:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695964.025139:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930157184 : -128555779394432 : ffff8b14466f7480) 00000100:00000040:1.0:1787695964.025147:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7480 x1874531139553024/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.025154:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695964.025158:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695964.025160:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695964.025163:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139553024, offset 0 00000400:00000200:1.0:1787695964.025167:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695964.025172:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695964.025176:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695964.025180:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695964.025182:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695964.025184:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695964.025187:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695964.025190:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17130:17130:256:4294967295] 192.168.201.140@tcp LPNI seq info [17129:17129:8:4294967295] 00000400:00000200:1.0:1787695964.025195:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695964.025202:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695964.025207:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695964.025210:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695964.025215:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695964.025219:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1787695964.025224:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695964.025233:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1787695964.025237:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695964.025242:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695964.025246:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695964.025251:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695964.025255:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695964.025257:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695964.025261:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695964.025265:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7480 x1874531139553024/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.025274:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695964.025280:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7480 x1874531139553024/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695989 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.025287:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.025290:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695964.025293:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695964.025296:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787695964.025300:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695964.025302:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298524487 00000100:00000001:1.0:1787695964.025304:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298535737 00000100:00000001:1.0:1787695964.025306:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298524487 00000100:00000001:1.0:1787695964.025309:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298535737 00000100:00000001:1.0:1787695964.025312:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695964.025315:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.025317:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695964.025322:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695964/real 1787695964] req@ffff8b14466f7480 x1874531139553024/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695964.025331:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695964.025332:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695964.025335:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695964.025337:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695964.025339:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695964.025351:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695964.025355:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7480 x1874531139553024/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695964.025391:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7480 x1874531139553024/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.025397:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695964.025400:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.025402:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695964.025403:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695964.025406:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7480 x1874531139553024/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.025412:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695964.025415:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7480 x1874531139553024/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.025419:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695964.025420:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.025421:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695964.025422:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.025423:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695964.025424:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695964.025426:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695964.025428:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695964.025430:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695964.025432:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7480 x1874531139553024/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695964.025437:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139553024:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695964.025440:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695964.025447:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6a00 x1874531139553152/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.025452:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695964.025453:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695964.025455:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695964.025456:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695964.025458:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695964.025460:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139553152:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695964.025462:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695964.025464:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695964.025465:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695964.025466:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695964.025468:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695964.025470:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695964.025473:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695964.025475:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695964.025478:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695964.025481:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139553152, portal 4 00000100:00000001:1.0:1787695964.025482:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695964.025483:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930154496 : -128555779397120 : ffff8b14466f6a00) 00000100:00000040:1.0:1787695964.025488:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6a00 x1874531139553152/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.025492:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695964.025494:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695964.025496:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695964.025497:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139553152, offset 0 00000400:00000200:1.0:1787695964.025500:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695964.025503:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695964.025507:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695964.025510:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695964.025513:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695964.025516:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695964.025518:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695964.025526:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17131:17131:256:4294967295] 192.168.201.140@tcp LPNI seq info [17130:17130:8:4294967295] 00000400:00000200:1.0:1787695964.025531:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695964.025539:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695964.025544:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695964.025548:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695964.025552:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695964.025556:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1787695964.025560:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695964.025564:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1787695964.025568:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695964.025570:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695964.025574:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695964.025577:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695964.025580:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695964.025582:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695964.025585:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695964.025589:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6a00 x1874531139553152/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.025598:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695964.025602:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6a00 x1874531139553152/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695989 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.025609:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.025611:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695964.025614:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695964.025618:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695964.025622:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695964.025624:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298524518 00000100:00000001:1.0:1787695964.025627:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298535737 00000100:00000001:1.0:1787695964.025629:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298524518 00000100:00000001:1.0:1787695964.025631:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298535737 00000100:00000001:1.0:1787695964.025634:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695964.025642:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.025644:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695964.025649:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695964/real 1787695964] req@ffff8b14466f6a00 x1874531139553152/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695964.025656:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695964.025657:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695964.025658:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695964.025660:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695964.025661:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695964.025663:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695964.025666:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6a00 x1874531139553152/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695964.025676:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6a00 x1874531139553152/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.025684:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695964.025686:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.025687:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695964.025689:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695964.025692:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6a00 x1874531139553152/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.025699:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695964.025704:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6a00 x1874531139553152/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.025711:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695964.025714:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.025715:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695964.025717:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.025719:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695964.025720:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695964.025722:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695964.025730:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695964.025731:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695964.025735:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6a00 x1874531139553152/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695964.025744:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139553152:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695964.025748:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.025750:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695964.025753:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4e00 x1874531139552896/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.025761:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695964.025762:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695964.025765:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695964.025769:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695964.025771:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695964.025773:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695964.025775:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695964.025777:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:1.0:1787695964.025783:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695964.025784:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695964.025786:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4e00. 00000100:00000001:1.0:1787695964.025789:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695964.025791:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.025792:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695964.025795:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7480 x1874531139553024/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.025800:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695964.025801:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695964.025802:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695964.025804:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695964.025804:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695964.025805:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695964.025806:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695964.025812:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676000. 02000000:00000001:1.0:1787695964.025815:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695964.025816:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695964.025818:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7480. 00000100:00000001:1.0:1787695964.025820:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695964.025822:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.025824:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695964.025827:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6a00 x1874531139553152/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.025834:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695964.025836:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695964.025837:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695964.025840:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695964.025842:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695964.025844:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695964.025846:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695964.025847:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:1.0:1787695964.025850:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695964.025851:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695964.025853:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6a00. 00000100:00000001:1.0:1787695964.025856:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695964.025857:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.025859:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.025864:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695964.025866:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.025868:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695964.025870:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695964.025871:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695964.025876:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5c00 x1874531139553280/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.025883:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695964.025885:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695964.025886:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695964.025888:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695964.025890:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695964.025894:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139553280:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695964.025903:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695964.025906:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695964.025908:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695964.025910:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695964.025913:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695964.025915:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695964.025919:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695964.025922:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695964.025925:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695964.025929:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139553280, portal 4 00000100:00000001:1.0:1787695964.025931:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695964.025932:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150912 : -128555779400704 : ffff8b14466f5c00) 00000100:00000040:1.0:1787695964.025938:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5c00 x1874531139553280/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.025944:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695964.025947:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695964.025949:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695964.025951:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139553280, offset 0 00000400:00000200:1.0:1787695964.025956:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695964.025960:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695964.025964:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695964.025967:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695964.025969:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695964.025972:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695964.025974:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695964.025977:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17132:17132:256:4294967295] 192.168.201.140@tcp LPNI seq info [17131:17131:8:4294967295] 00000400:00000200:1.0:1787695964.025981:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695964.025987:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695964.025992:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695964.025996:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695964.026001:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695964.026011:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1787695964.026016:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695964.026018:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1787695964.026022:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695964.026024:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695964.026027:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695964.026032:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695964.026036:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695964.026037:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695964.026040:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695964.026223:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5c00 x1874531139553280/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.026230:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695964.026232:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5c00 x1874531139553280/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695989 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.026236:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.026237:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695964.026242:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695964.026245:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695964.026249:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695964.026252:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298524190 00000100:00000001:1.0:1787695964.026255:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298535737 00000100:00000001:1.0:1787695964.026257:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298524190 00000100:00000001:1.0:1787695964.026259:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298535737 00000100:00000001:1.0:1787695964.026261:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695964.026264:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.026266:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695964.026271:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695964/real 1787695964] req@ffff8b14466f5c00 x1874531139553280/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695964.026280:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695964.026282:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695964.026290:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695964.026292:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695964.026294:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695964.026297:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695964.026300:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5c00 x1874531139553280/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695964.026306:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5c00 x1874531139553280/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.026311:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695964.026312:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.026314:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695964.026316:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695964.026320:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5c00 x1874531139553280/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.026327:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695964.026331:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5c00 x1874531139553280/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.026339:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695964.026342:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.026343:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695964.026345:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.026347:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695964.026349:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695964.026352:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695964.026356:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695964.026358:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695964.026403:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5c00 x1874531139553280/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695964.026415:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139553280:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695964.026426:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695964.026431:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7800 x1874531139553408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.026439:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695964.026440:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695964.026442:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695964.026445:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695964.026447:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695964.026451:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139553408:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695964.026454:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695964.026457:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695964.026458:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695964.026459:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695964.026462:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695964.026464:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695964.026466:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695964.026468:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695964.026471:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695964.026474:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139553408, portal 10 00000100:00000001:1.0:1787695964.026475:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695964.026476:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930158080 : -128555779393536 : ffff8b14466f7800) 00000100:00000040:1.0:1787695964.026482:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7800 x1874531139553408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.026487:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695964.026488:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695964.026490:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695964.026491:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139553408, offset 0 00000400:00000200:1.0:1787695964.026494:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695964.026498:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695964.026501:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695964.026504:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695964.026513:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695964.026516:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695964.026518:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695964.026522:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17133:17133:256:4294967295] 192.168.201.140@tcp LPNI seq info [17132:17132:8:4294967295] 00000400:00000200:1.0:1787695964.026527:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695964.026534:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695964.026539:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695964.026543:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695964.026548:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695964.026551:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1787695964.026556:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695964.026559:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1787695964.026562:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695964.026564:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695964.026566:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695964.026569:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695964.026571:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695964.026573:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695964.026575:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695964.026579:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7800 x1874531139553408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.026587:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695964.026591:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7800 x1874531139553408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695989 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.026598:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.026599:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695964.026603:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695964.026607:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695964.026611:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695964.026613:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298524377 00000100:00000001:1.0:1787695964.026615:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298535737 00000100:00000001:1.0:1787695964.026618:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298524377 00000100:00000001:1.0:1787695964.026626:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298535737 00000100:00000001:1.0:1787695964.026628:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695964.026631:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.026633:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695964.026638:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695964/real 1787695964] req@ffff8b14466f7800 x1874531139553408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695964.026647:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695964.026648:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695964.026650:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695964.026653:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695964.026655:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695964.026657:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695964.026661:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7800 x1874531139553408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695964.026670:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7800 x1874531139553408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.026677:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695964.026679:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.026681:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695964.026682:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695964.026686:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7800 x1874531139553408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.026692:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695964.026696:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7800 x1874531139553408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.026703:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695964.026706:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.026707:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695964.026709:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.026717:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695964.026719:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695964.026722:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695964.026726:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695964.026729:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695964.026733:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7800 x1874531139553408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695964.026741:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139553408:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695964.026746:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695964.026750:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7b80 x1874531139553536/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.026757:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695964.026758:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695964.026760:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695964.026763:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695964.026765:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695964.026768:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139553536:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695964.026772:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695964.026774:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695964.026776:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695964.026778:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695964.026782:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695964.026785:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695964.026789:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695964.026792:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695964.026795:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695964.026799:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139553536, portal 10 00000100:00000001:1.0:1787695964.026801:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695964.026802:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930158976 : -128555779392640 : ffff8b14466f7b80) 00000100:00000040:1.0:1787695964.026809:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7b80 x1874531139553536/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.026822:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695964.026825:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695964.026828:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695964.026831:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139553536, offset 0 00000400:00000200:1.0:1787695964.026836:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695964.026840:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695964.026844:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695964.026848:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695964.026851:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695964.026854:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695964.026856:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695964.026860:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17134:17134:256:4294967295] 192.168.201.140@tcp LPNI seq info [17133:17133:8:4294967295] 00000400:00000200:1.0:1787695964.026864:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695964.026869:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695964.026872:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695964.026875:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695964.026878:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695964.026880:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 38 secs later) 00000800:00000200:1.1:1787695964.026883:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695964.026885:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 38 secs later) 00000800:00000100:1.0:1787695964.026888:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695964.026889:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695964.026892:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695964.026894:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695964.026896:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695964.026898:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695964.026900:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695964.026903:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7b80 x1874531139553536/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695989 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.026912:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695964.026915:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7b80 x1874531139553536/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695989 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.026936:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.026938:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695964.026942:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695964.026946:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695964.026950:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695964.026952:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298524355 00000100:00000001:1.0:1787695964.026955:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298535737 00000100:00000001:1.0:1787695964.026958:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298524355 00000100:00000001:1.0:1787695964.026960:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298535737 00000100:00000001:1.0:1787695964.026962:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695964.026963:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.026965:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695964.026968:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695964/real 1787695964] req@ffff8b14466f7b80 x1874531139553536/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695964.026976:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695964.026978:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695964.026980:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695964.026982:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695964.026984:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695964.026987:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695964.026990:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7b80 x1874531139553536/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695964.026999:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7b80 x1874531139553536/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.027005:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695964.027007:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.027009:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695964.027011:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695964.027015:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7b80 x1874531139553536/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.027028:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695964.027032:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7b80 x1874531139553536/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.027039:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695964.027041:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.027056:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695964.027058:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.027060:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695964.027062:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695964.027064:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695964.027069:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695964.027071:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695964.027075:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7b80 x1874531139553536/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695964.027083:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139553536:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695964.027087:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.027090:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695964.027093:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5c00 x1874531139553280/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.027099:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695964.027101:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695964.027103:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695964.027106:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695964.027108:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695964.027109:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695964.027111:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695964.027113:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:1.0:1787695964.027122:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695964.027124:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695964.027126:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5c00. 00000100:00000001:1.0:1787695964.027128:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695964.027130:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.027132:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695964.027136:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7800 x1874531139553408/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.027143:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695964.027145:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695964.027147:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695964.027150:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695964.027151:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695964.027153:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695964.027155:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695964.027156:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675800. 02000000:00000001:1.0:1787695964.027161:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695964.027162:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695964.027163:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7800. 00000100:00000001:1.0:1787695964.027165:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695964.027167:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.027168:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695964.027172:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7b80 x1874531139553536/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695989 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695964.027178:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695964.027180:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695964.027182:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695964.027184:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695964.027185:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695964.027187:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695964.027189:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695964.027190:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:1.0:1787695964.027193:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695964.027194:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695964.027196:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7b80. 00000100:00000001:1.0:1787695964.027198:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695964.027204:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.027207:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695964.027212:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695964.027214:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695964.027216:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695964.027219:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695964.086543:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695964.086556:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695964.086558:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695964.086566:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695964.086571:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695964.086592:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695964.086598:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695964.086602:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695964.086606:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695964.726517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695964.726527:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695964.726536:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695964.726538:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695964.726542:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695964.726543:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695964.726547:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695964.726548:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695964.726550:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695964.726551:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695964.726554:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695964.726555:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695964.726626:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695964.726631:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695964.726635:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695965.110448:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695965.110454:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695965.110456:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695965.110460:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695965.110462:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695965.110465:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695965.110467:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695965.110469:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695965.110471:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695965.750521:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695965.750528:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695965.750534:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695965.750535:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695965.750538:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695965.750539:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695965.750543:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695965.750543:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695965.750546:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695965.750547:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695965.750550:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695965.750551:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695965.750639:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695965.750645:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695965.750649:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695966.134480:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695966.134486:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695966.134487:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695966.134492:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695966.134512:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695966.134515:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695966.134517:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695966.134519:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695966.134521:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695966.774517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695966.774526:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695966.774534:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695966.774535:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695966.774540:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695966.774544:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695966.774547:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695966.774555:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695966.774558:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695966.774583:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695966.774586:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695966.774589:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695966.774591:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695966.774594:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695966.774599:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695966.774601:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695966.774602:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695966.774605:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695966.774606:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695966.774607:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695966.774609:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695966.774610:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695966.774612:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695966.774614:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695966.774619:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695966.774620:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695966.774624:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695966.774626:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695966.774628:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695966.774630:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695966.774631:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695966.774632:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695966.774634:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695966.774636:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695966.774638:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695966.774641:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695966.774729:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695966.774736:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695966.774739:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695967.158492:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695967.158499:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695967.158501:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695967.158507:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695967.158512:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695967.158515:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695967.158518:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695967.158521:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695967.158524:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695967.798491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695967.798497:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695967.798501:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695967.798502:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695967.798505:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695967.798506:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695967.798508:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695967.798509:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695967.798512:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695967.798513:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695967.798515:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695967.798515:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695967.798595:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695967.798600:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695967.798603:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695968.182452:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695968.182461:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695968.182463:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695968.182469:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695968.182471:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695968.182474:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695968.182476:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695968.182479:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695968.182480:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695968.822481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695968.822487:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695968.822491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695968.822492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695968.822494:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695968.822495:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695968.822497:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695968.822498:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695968.822501:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695968.822501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695968.822504:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695968.822505:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695968.822565:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695968.822569:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695968.822571:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787695969.142448:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695969.142460:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787695969.142463:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695969.142467:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695969.142472:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695969.142478:0:47074:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3856 00000400:00000200:3.0:1787695969.142486:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695969.142492:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695969.142495:0:47074:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695969.142498:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.142512:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695969.142514:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.142516:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695969.142519:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695969.142521:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:3.0:1787695969.142523:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695969.142525:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787695969.142527:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787695969.142529:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787695969.142531:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.142532:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695969.142533:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.142534:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695969.142536:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787695969.142539:0:47074:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695969.142540:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695969.142542:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695969.142545:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695969.142548:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695969.142551:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787695969.142554:0:47074:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695969.142560:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4a80. 00000020:00000040:3.0:1787695969.142563:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787695969.142567:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695969.142568:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695969.142571:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695969.142574:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695969.142578:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695969.142586:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675400. 00000100:00000001:3.0:1787695969.142592:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695969.142603:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4a80 x1874531139553664/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695969.142630:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695969.142633:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695969.142638:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ad0d67] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4a80 x1874531139553664/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695969.142652:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695969.142655:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695969.142657:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695969.142658:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695969.142660:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695969.142662:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695969.142664:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3856 00000400:00000200:3.0:1787695969.142667:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695969.142669:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695969.142671:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695969.142673:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.142675:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695969.142677:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.142679:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695969.142681:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695969.142682:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:3.0:1787695969.142683:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695969.142685:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:3.0:1787695969.142686:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000001:1.0:1787695969.142687:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787695969.142688:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:3.0:1787695969.142693:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.142694:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695969.142695:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.142697:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695969.142699:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000040:3.0:1787695969.142705:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00000001:1.0:1787695969.142706:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1787695969.142709:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695969.142713:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695969.142715:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4a80 x1874531139553664/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695969.142715:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695969.142720:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695969.142725:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.142728:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695969.142728:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695969.142732:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787695969.142734:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695969.142737:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695969.142738:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787695969.142739:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695969.142744:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:3.0:1787695969.142745:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6680. 00000020:00000040:3.0:1787695969.142748:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:1.0:1787695969.142752:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695969.142752:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695969.142754:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695969.142757:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695969.142760:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1787695969.142761:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139553664:192.168.201.140@tcp:250:kworker.0 02000000:00000001:3.0:1787695969.142762:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.142770:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000010:3.0:1787695969.142770:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 00000100:00000001:3.0:1787695969.142775:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695969.142777:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695969.142782:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695969.142783:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6680 x1874531139553792/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787695969.142788:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787695969.142794:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000010:1.0:1787695969.142795:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 00000100:00000001:3.0:1787695969.142796:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695969.142800:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695969.142804:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f6d2abf9] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6680 x1874531139553792/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695969.142809:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695969.142815:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695969.142823:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000001:3.0:1787695969.142824:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695969.142830:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139553664, portal 25 00000100:00000001:1.0:1787695969.142833:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695969.142835:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930146432 : -128555779405184 : ffff8b14466f4a80) 00000100:00080000:3.0:1787695969.142843:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1787695969.142848:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4a80 x1874531139553664/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695969.142850:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695969.142855:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695969.142858:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695969.142860:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695969.142865:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695969.142866:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:1.0:1787695969.142867:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695969.142871:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139553664, offset 0 00000400:00000200:1.0:1787695969.142876:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00080000:3.0:1787695969.142882:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3856 00000400:00000200:1.0:1787695969.142886:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695969.142890:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695969.142896:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:3.0:1787695969.142897:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695969.142903:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000400:00000200:1.0:1787695969.142906:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1787695969.142907:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.142910:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695969.142912:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000040:3.0:1787695969.142915:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.142918:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695969.142919:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000020:00000001:3.0:1787695969.142921:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695969.142923:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000400:00000200:1.0:1787695969.142924:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000020:00000001:3.0:1787695969.142926:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:1.0:1787695969.142930:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17135:17135:256:4294967295] 192.168.201.140@tcp LPNI seq info [17134:17134:8:4294967295] 00000020:00000040:3.0:1787695969.142930:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:3.0:1787695969.142937:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000400:00000200:1.0:1787695969.142939:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:3.0:1787695969.142945:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:3.0:1787695969.142967:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695969.142968:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:3.0:1787695969.142973:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:1.0:1787695969.142980:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695969.142981:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.142986:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000010:1.0:1787695969.142987:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000020:00000040:3.0:1787695969.142993:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000800:00000200:1.0:1787695969.142998:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00080000:3.0:1787695969.143003:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000800:00000200:1.0:1787695969.143004:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 33 secs later) 00000100:00000001:3.0:1787695969.143011:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1787695969.143013:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:3.0:1787695969.143017:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000200:1.1:1787695969.143018:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 33 secs later) 02000000:00000001:3.0:1787695969.143021:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000100:1.0:1787695969.143023:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695969.143027:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 02000000:00000001:3.0:1787695969.143027:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695969.143031:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:1.0:1787695969.143033:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000008:00100000:3.0:1787695969.143034:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695969.143039:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695969.143042:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1787695969.143043:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695969.143048:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000100:00000010:3.0:1787695969.143049:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5880. 00000400:00000010:1.0:1787695969.143052:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000020:00000040:3.0:1787695969.143053:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000100:00000001:1.0:1787695969.143057:0:2402:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1787695969.143058:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695969.143064:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695969.143066:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695969.143068:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:1.0:1787695969.143070:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4a80 x1874531139553664/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695969.143073:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695969.143082:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675800. 00000100:00000001:1.0:1787695969.143083:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787695969.143088:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695969.143095:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4a80 x1874531139553664/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695994 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695969.143100:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5880 x1874531139553920/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695969.143105:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.143109:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695969.143117:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695969.143117:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695969.143122:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695969.143125:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695969.143129:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298540856 00000100:00000040:3.0:1787695969.143131:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000827e7dbe] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5880 x1874531139553920/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.143136:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298535737 00000100:00000001:1.0:1787695969.143142:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298540856 00000100:00000001:1.0:1787695969.143149:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298535737 00000100:00000001:3.0:1787695969.143149:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.143153:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1787695969.143155:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695969.143156:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.143161:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1787695969.143163:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787695969.143167:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695969/real 1787695969] req@ffff8b14466f4a80 x1874531139553664/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695969.143168:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695969.143172:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:1.0:1787695969.143183:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695969.143184:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695969.143186:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000100:00000001:3.0:1787695969.143187:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1787695969.143188:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695969.143192:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695969.143194:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:3.0:1787695969.143198:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3856 00000100:00000200:1.0:1787695969.143199:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4a80 x1874531139553664/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695969.143210:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000200:1.0:1787695969.143211:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4a80 x1874531139553664/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.143217:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695969.143218:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.143220:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695969.143221:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695969.143228:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4a80 x1874531139553664/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695969.143228:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695969.143234:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695969.143241:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.143245:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695969.143247:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.143248:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695969.143251:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695969.143253:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:3.0:1787695969.143256:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695969.143259:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:3.0:1787695969.143263:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:1.0:1787695969.143268:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787695969.143273:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:1.0:1787695969.143278:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4a80 x1874531139553664/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695969.143279:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.143284:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695969.143287:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.143292:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695969.143297:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00000001:1.0:1787695969.143301:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787695969.143302:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695969.143309:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.143310:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787695969.143310:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695969.143315:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1787695969.143315:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695969.143319:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787695969.143320:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695969.143325:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695969.143326:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000008:00100000:3.0:1787695969.143327:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787695969.143329:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:3.0:1787695969.143330:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695969.143334:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695969.143335:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000010:3.0:1787695969.143337:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4000. 00000020:00000040:3.0:1787695969.143341:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000100:00080000:1.0:1787695969.143343:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 02000000:00000001:3.0:1787695969.143344:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695969.143346:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.143350:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787695969.143350:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695969.143352:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695969.143354:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695969.143359:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 00000100:00000040:1.0:1787695969.143361:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4a80 x1874531139553664/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695969.143383:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695969.143389:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4000 x1874531139554048/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695969.143395:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695969.143397:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695969.143400:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5a8f356] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4000 x1874531139554048/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695969.143434:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695969.143437:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139553664:192.168.201.140@tcp:250:kworker.0 00000100:00000001:1.0:1787695969.143443:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787695969.143445:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695969.143448:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1787695969.143452:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000040:1.0:1787695969.143454:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4a80 x1874531139553664/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695969.143456:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695969.143460:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695969.143468:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695969.143471:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00080000:3.0:1787695969.143475:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3856 02000000:00000001:1.0:1787695969.143476:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695969.143481:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 00000400:00000200:3.0:1787695969.143482:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695969.143488:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 02000000:00000001:1.0:1787695969.143489:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:3.0:1787695969.143490:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000020:00000001:1.0:1787695969.143494:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000040:3.0:1787695969.143495:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.143498:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000040:1.0:1787695969.143499:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000100:00000040:3.0:1787695969.143501:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000020:00000001:1.0:1787695969.143505:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:3.0:1787695969.143505:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695969.143509:0:47074:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000010:1.0:1787695969.143510:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675400. 00000020:00000040:3.0:1787695969.143511:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:3.0:1787695969.143515:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695969.143516:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 02000000:00000001:1.0:1787695969.143518:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:3.0:1787695969.143518:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:3.0:1787695969.143520:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:3.0:1787695969.143522:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.143523:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695969.143526:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695969.143528:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787695969.143529:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695969.143533:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:3.0:1787695969.143536:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695969.143537:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695969.143538:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695969.143541:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000010:1.0:1787695969.143542:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4a80. 02000000:00000001:3.0:1787695969.143543:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695969.143548:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695969.143552:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695969.143554:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.143560:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1787695969.143560:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787695969.143567:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695969.143569:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.143571:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:3.0:1787695969.143574:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4700. 00000100:00000001:1.0:1787695969.143575:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787695969.143581:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 00000100:00000001:1.0:1787695969.143582:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695969.143585:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695969.143588:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787695969.143588:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695969.143593:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695969.143597:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6680 x1874531139553792/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695969.143598:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695969.143601:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695969.143604:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695969.143610:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 00000100:00000001:1.0:1787695969.143613:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695969.143615:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.143616:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695969.143622:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695969.143626:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1787695969.143626:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4700 x1874531139554176/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787695969.143633:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695969.143639:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1787695969.143641:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139553792:192.168.201.140@tcp:38:kworker.0 00000100:00000001:3.0:1787695969.143642:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.143649:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:3.0:1787695969.143650:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008818c2cc] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4700 x1874531139554176/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695969.143653:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695969.143656:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695969.143659:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695969.143661:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1787695969.143663:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695969.143666:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 02000000:00000010:1.0:1787695969.143667:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 00000100:00000001:3.0:1787695969.143668:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695969.143671:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1787695969.143672:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695969.143675:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695969.143679:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3856 00000400:00000200:1.0:1787695969.143680:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695969.143686:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695969.143687:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000200:3.0:1787695969.143690:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695969.143693:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000400:00000010:1.0:1787695969.143694:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000040:3.0:1787695969.143696:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000200:1.0:1787695969.143700:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139553792, portal 10 00000100:00000001:3.0:1787695969.143700:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695969.143704:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695969.143706:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695969.143707:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930153600 : -128555779398016 : ffff8b14466f6680) 00000100:00000001:3.0:1787695969.143713:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695969.143716:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000040:1.0:1787695969.143717:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6680 x1874531139553792/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1787695969.143718:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:3.0:1787695969.143722:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695969.143725:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000020:00000001:3.0:1787695969.143729:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000020:00000001:3.0:1787695969.143733:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:1.0:1787695969.143736:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1787695969.143740:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.143744:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695969.143745:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695969.143747:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.143748:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695969.143750:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:3.0:1787695969.143752:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695969.143756:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695969.143759:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 02000000:00000001:3.0:1787695969.143759:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695969.143762:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000200:1.0:1787695969.143766:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139553792, offset 0 02000000:00000001:3.0:1787695969.143766:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695969.143770:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695969.143773:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695969.143777:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695969.143778:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695969.143779:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695969.143786:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5500. 00000400:00000200:1.0:1787695969.143790:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000040:3.0:1787695969.143791:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:3.0:1787695969.143795:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695969.143797:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695969.143799:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:3.0:1787695969.143800:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1787695969.143805:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:3.0:1787695969.143805:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695969.143809:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695969.143811:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695969.143816:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 02000000:00000010:3.0:1787695969.143818:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676000. 00000400:00000200:1.0:1787695969.143820:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.143825:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695969.143826:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17136:17136:256:4294967295] 192.168.201.140@tcp LPNI seq info [17135:17135:8:4294967295] 00000400:00000200:1.0:1787695969.143836:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00100000:3.0:1787695969.143837:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5500 x1874531139554304/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695969.143851:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695969.143858:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695969.143865:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000100:00000001:3.0:1787695969.143869:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695969.143873:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695969.143876:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:3.0:1787695969.143880:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a777d68] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5500 x1874531139554304/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1787695969.143881:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1787695969.143893:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787695969.143896:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1787695969.143897:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 33 secs later) 00000100:00080000:3.0:1787695969.143901:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787695969.143905:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00080000:3.0:1787695969.143908:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000010:1.0:1787695969.143909:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000100:00000001:3.0:1787695969.143912:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1787695969.143914:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:3.0:1787695969.143915:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695969.143921:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1787695969.143922:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00080000:3.0:1787695969.143926:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3856 00000400:00000200:1.0:1787695969.143927:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695969.143931:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000400:00000200:3.0:1787695969.143934:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695969.143936:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:3.0:1787695969.143939:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000200:1.0:1787695969.143943:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6680 x1874531139553792/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695969.143943:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695969.143950:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.143968:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695969.143973:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695969.143976:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787695969.143976:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695969.143979:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000040:1.0:1787695969.143981:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6680 x1874531139553792/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695994 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1787695969.143982:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000020:00000001:3.0:1787695969.143985:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695969.143988:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:3.0:1787695969.143993:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:1.0:1787695969.143994:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695969.143996:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:1.0:1787695969.143997:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1787695969.144001:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.144003:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695969.144004:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695969.144008:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695969.144011:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695969.144015:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000020:00000040:3.0:1787695969.144021:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000400:00000200:1.0:1787695969.144022:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:3.0:1787695969.144026:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695969.144027:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298530758 00000100:00000001:3.0:1787695969.144031:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.144032:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298540856 02000000:00000001:3.0:1787695969.144035:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695969.144036:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298530758 02000000:00000001:3.0:1787695969.144039:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695969.144040:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298540856 00000100:00000001:1.0:1787695969.144046:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695969.144046:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.144051:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695969.144051:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695969.144055:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000008:00100000:3.0:1787695969.144055:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695969.144060:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695969.144062:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695969/real 1787695969] req@ffff8b14466f6680 x1874531139553792/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695969.144063:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695969.144068:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5f80. 00000020:00000040:3.0:1787695969.144073:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000400:00000001:1.0:1787695969.144074:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1787695969.144076:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000001:1.0:1787695969.144077:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1787695969.144080:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695969.144081:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000100:00000001:3.0:1787695969.144083:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1787695969.144085:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 02000000:00000001:3.0:1787695969.144086:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1787695969.144088:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 02000000:00000001:3.0:1787695969.144089:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.144091:0:2402:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000010:3.0:1787695969.144094:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 00000100:00000200:1.0:1787695969.144096:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6680 x1874531139553792/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695969.144099:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695969.144108:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5f80 x1874531139554432/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:1.0:1787695969.144110:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6680 x1874531139553792/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695969.144128:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695969.144130:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787695969.144131:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.144133:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.144136:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695969.144139:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695969.144140:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006e7bde76] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5f80 x1874531139554432/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695969.144144:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6680 x1874531139553792/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695969.144160:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.144162:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695969.144164:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1787695969.144168:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6680 x1874531139553792/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695969.144169:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695969.144172:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787695969.144177:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1787695969.144182:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695969.144188:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.144191:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695969.144192:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695969.144192:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3866) 00000100:00000001:1.0:1787695969.144197:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695969.144199:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695969.144201:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695969.144204:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695969.144206:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695969.144209:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6680 x1874531139553792/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695969.144213:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139553792:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695969.144216:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695969.144218:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5880 x1874531139553920/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144222:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695969.144223:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695969.144224:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695969.144226:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695969.144227:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695969.144229:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139553920:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695969.144231:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695969.144232:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695969.144233:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695969.144234:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695969.144237:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695969.144238:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695969.144244:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695969.144246:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695969.144249:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695969.144251:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139553920, portal 10 00000100:00000001:1.0:1787695969.144253:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695969.144254:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150016 : -128555779401600 : ffff8b14466f5880) 00000100:00000040:1.0:1787695969.144258:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5880 x1874531139553920/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144262:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695969.144264:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695969.144265:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695969.144267:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139553920, offset 0 00000400:00000200:1.0:1787695969.144270:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695969.144272:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695969.144275:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695969.144277:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695969.144279:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695969.144281:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695969.144282:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695969.144284:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17137:17137:256:4294967295] 192.168.201.140@tcp LPNI seq info [17136:17136:8:4294967295] 00000400:00000200:1.0:1787695969.144287:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695969.144292:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695969.144295:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695969.144297:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695969.144299:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695969.144301:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1787695969.144304:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695969.144306:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1787695969.144309:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695969.144310:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695969.144312:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695969.144314:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695969.144317:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695969.144321:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695969.144323:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695969.144325:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5880 x1874531139553920/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144332:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695969.144336:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5880 x1874531139553920/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695994 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144343:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.144345:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695969.144348:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695969.144351:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787695969.144354:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695969.144357:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298529606 00000100:00000001:1.0:1787695969.144360:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298540856 00000100:00000001:1.0:1787695969.144362:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298529606 00000100:00000001:1.0:1787695969.144382:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298540856 00000100:00000001:1.0:1787695969.144385:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695969.144387:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.144389:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695969.144394:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695969/real 1787695969] req@ffff8b14466f5880 x1874531139553920/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695969.144402:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695969.144403:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695969.144405:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695969.144407:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695969.144409:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695969.144412:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695969.144415:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5880 x1874531139553920/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695969.144424:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5880 x1874531139553920/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144437:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695969.144438:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.144440:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695969.144441:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695969.144444:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5880 x1874531139553920/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144449:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695969.144451:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5880 x1874531139553920/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144456:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695969.144457:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.144458:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695969.144459:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.144460:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695969.144461:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695969.144463:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695969.144465:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695969.144466:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695969.144468:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5880 x1874531139553920/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695969.144473:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139553920:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695969.144475:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695969.144477:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4000 x1874531139554048/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144482:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695969.144483:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695969.144488:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695969.144489:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695969.144490:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695969.144492:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139554048:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695969.144495:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695969.144496:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695969.144497:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695969.144499:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695969.144501:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695969.144502:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695969.144505:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695969.144507:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695969.144510:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695969.144512:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139554048, portal 4 00000100:00000001:1.0:1787695969.144514:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695969.144515:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930143744 : -128555779407872 : ffff8b14466f4000) 00000100:00000040:1.0:1787695969.144519:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4000 x1874531139554048/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144523:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695969.144525:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695969.144526:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695969.144528:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139554048, offset 0 00000400:00000200:1.0:1787695969.144530:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695969.144533:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695969.144535:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695969.144537:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695969.144539:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695969.144541:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695969.144543:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695969.144545:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17138:17138:256:4294967295] 192.168.201.140@tcp LPNI seq info [17137:17137:8:4294967295] 00000400:00000200:1.0:1787695969.144548:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695969.144552:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695969.144559:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695969.144561:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695969.144564:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695969.144566:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1787695969.144569:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695969.144571:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1787695969.144573:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695969.144575:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695969.144577:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695969.144579:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695969.144582:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695969.144583:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695969.144584:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695969.144587:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4000 x1874531139554048/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144592:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695969.144594:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4000 x1874531139554048/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695994 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144598:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.144599:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695969.144601:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695969.144603:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695969.144605:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695969.144607:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298529637 00000100:00000001:1.0:1787695969.144609:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298540856 00000100:00000001:1.0:1787695969.144610:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298529637 00000100:00000001:1.0:1787695969.144611:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298540856 00000100:00000001:1.0:1787695969.144613:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695969.144614:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.144615:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695969.144617:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695969/real 1787695969] req@ffff8b14466f4000 x1874531139554048/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695969.144627:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695969.144628:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695969.144630:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695969.144631:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695969.144632:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695969.144634:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695969.144636:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4000 x1874531139554048/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695969.144641:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4000 x1874531139554048/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144646:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695969.144647:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.144648:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695969.144648:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695969.144650:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4000 x1874531139554048/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144654:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695969.144657:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4000 x1874531139554048/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144661:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695969.144662:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.144663:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695969.144664:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.144665:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695969.144666:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695969.144667:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695969.144669:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695969.144671:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695969.144676:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4000 x1874531139554048/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695969.144682:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139554048:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695969.144684:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.144685:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695969.144687:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6680 x1874531139553792/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144691:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695969.144692:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695969.144694:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695969.144696:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695969.144697:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695969.144698:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695969.144699:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695969.144701:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:1.0:1787695969.144704:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695969.144705:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695969.144706:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6680. 00000100:00000001:1.0:1787695969.144708:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695969.144709:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.144710:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695969.144712:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5880 x1874531139553920/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144716:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695969.144717:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695969.144718:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695969.144719:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695969.144719:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695969.144720:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695969.144721:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695969.144722:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675800. 02000000:00000001:1.0:1787695969.144723:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695969.144727:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695969.144728:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5880. 00000100:00000001:1.0:1787695969.144729:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695969.144730:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.144731:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695969.144734:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4000 x1874531139554048/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144737:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695969.144738:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695969.144739:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695969.144741:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695969.144742:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695969.144742:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695969.144743:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695969.144744:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:1.0:1787695969.144747:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695969.144747:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695969.144748:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4000. 00000100:00000001:1.0:1787695969.144749:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695969.144750:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.144751:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.144754:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695969.144755:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.144756:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695969.144757:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695969.144758:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695969.144760:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4700 x1874531139554176/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144764:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695969.144764:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695969.144766:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695969.144767:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695969.144768:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695969.144770:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139554176:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695969.144772:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695969.144776:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695969.144777:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695969.144779:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695969.144780:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695969.144782:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695969.144784:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695969.144786:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695969.144788:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695969.144790:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139554176, portal 4 00000100:00000001:1.0:1787695969.144791:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695969.144792:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930145536 : -128555779406080 : ffff8b14466f4700) 00000100:00000040:1.0:1787695969.144796:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4700 x1874531139554176/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144801:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695969.144802:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695969.144804:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695969.144805:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139554176, offset 0 00000400:00000200:1.0:1787695969.144808:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695969.144810:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695969.144812:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695969.144814:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695969.144816:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695969.144818:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695969.144819:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695969.144821:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17139:17139:256:4294967295] 192.168.201.140@tcp LPNI seq info [17138:17138:8:4294967295] 00000400:00000200:1.0:1787695969.144824:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695969.144828:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695969.144830:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695969.144833:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695969.144835:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695969.144837:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1787695969.144840:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695969.144842:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1787695969.144844:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695969.144849:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695969.144851:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695969.144853:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695969.144855:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695969.144856:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695969.144858:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695969.144860:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4700 x1874531139554176/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144865:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695969.144867:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4700 x1874531139554176/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695994 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144872:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.144873:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695969.144875:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695969.144877:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695969.144879:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695969.144880:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298529309 00000100:00000001:1.0:1787695969.144882:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298540856 00000100:00000001:1.0:1787695969.144884:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298529309 00000100:00000001:1.0:1787695969.144885:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298540856 00000100:00000001:1.0:1787695969.144887:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695969.144888:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.144889:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695969.144891:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695969/real 1787695969] req@ffff8b14466f4700 x1874531139554176/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695969.144896:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695969.144896:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695969.144897:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695969.144899:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695969.144900:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695969.144904:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695969.144907:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4700 x1874531139554176/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695969.144912:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4700 x1874531139554176/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144916:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695969.144917:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.144918:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695969.144919:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695969.144921:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4700 x1874531139554176/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144925:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695969.144928:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4700 x1874531139554176/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144932:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695969.144934:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.144935:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695969.144936:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.144937:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695969.144938:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695969.144939:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695969.144941:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695969.144942:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695969.144945:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4700 x1874531139554176/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695969.144949:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139554176:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695969.144952:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695969.144966:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5500 x1874531139554304/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.144976:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695969.144977:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695969.144978:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695969.144980:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695969.144981:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695969.144983:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139554304:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695969.144985:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695969.144987:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695969.144988:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695969.144989:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695969.144991:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695969.144992:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695969.144995:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695969.144997:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695969.144999:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695969.145001:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139554304, portal 10 00000100:00000001:1.0:1787695969.145002:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695969.145003:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930149120 : -128555779402496 : ffff8b14466f5500) 00000100:00000040:1.0:1787695969.145008:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5500 x1874531139554304/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.145012:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695969.145014:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695969.145015:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695969.145016:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139554304, offset 0 00000400:00000200:1.0:1787695969.145019:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695969.145021:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695969.145024:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695969.145025:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695969.145027:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695969.145029:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695969.145030:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695969.145037:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17140:17140:256:4294967295] 192.168.201.140@tcp LPNI seq info [17139:17139:8:4294967295] 00000400:00000200:1.0:1787695969.145039:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695969.145044:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695969.145047:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695969.145049:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695969.145052:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695969.145054:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1787695969.145056:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695969.145058:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1787695969.145061:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695969.145062:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695969.145064:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695969.145067:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695969.145069:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695969.145070:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695969.145072:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695969.145074:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5500 x1874531139554304/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.145079:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695969.145081:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5500 x1874531139554304/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695994 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.145086:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.145087:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695969.145089:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695969.145091:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695969.145093:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695969.145094:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298529495 00000100:00000001:1.0:1787695969.145096:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298540856 00000100:00000001:1.0:1787695969.145097:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298529495 00000100:00000001:1.0:1787695969.145099:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298540856 00000100:00000001:1.0:1787695969.145100:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695969.145104:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.145106:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695969.145108:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695969/real 1787695969] req@ffff8b14466f5500 x1874531139554304/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695969.145114:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695969.145115:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695969.145116:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695969.145117:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695969.145118:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695969.145120:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695969.145122:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5500 x1874531139554304/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695969.145126:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5500 x1874531139554304/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.145131:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695969.145131:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.145133:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695969.145134:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695969.145136:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5500 x1874531139554304/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.145140:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695969.145142:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5500 x1874531139554304/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.145146:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695969.145148:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.145148:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695969.145149:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.145150:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695969.145151:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695969.145153:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695969.145158:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695969.145159:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695969.145162:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5500 x1874531139554304/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695969.145167:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139554304:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695969.145170:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695969.145172:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5f80 x1874531139554432/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.145176:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695969.145177:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695969.145178:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695969.145179:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695969.145180:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695969.145182:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139554432:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695969.145184:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695969.145185:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695969.145186:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695969.145187:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695969.145189:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695969.145190:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695969.145192:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695969.145194:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695969.145195:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695969.145198:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139554432, portal 10 00000100:00000001:1.0:1787695969.145199:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695969.145200:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930151808 : -128555779399808 : ffff8b14466f5f80) 00000100:00000040:1.0:1787695969.145204:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5f80 x1874531139554432/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.145208:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695969.145212:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695969.145214:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695969.145216:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139554432, offset 0 00000400:00000200:1.0:1787695969.145218:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695969.145220:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695969.145223:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695969.145225:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695969.145226:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695969.145228:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695969.145229:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695969.145232:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17141:17141:256:4294967295] 192.168.201.140@tcp LPNI seq info [17140:17140:8:4294967295] 00000400:00000200:1.0:1787695969.145234:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695969.145238:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695969.145241:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695969.145242:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695969.145245:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695969.145247:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 33 secs later) 00000800:00000200:1.1:1787695969.145249:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695969.145251:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 33 secs later) 00000800:00000100:1.0:1787695969.145253:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695969.145255:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695969.145257:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695969.145259:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695969.145261:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695969.145262:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695969.145263:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695969.145266:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5f80 x1874531139554432/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695994 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.145271:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695969.145273:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5f80 x1874531139554432/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695994 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.145277:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.145278:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695969.145283:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695969.145286:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695969.145288:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695969.145289:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298529473 00000100:00000001:1.0:1787695969.145291:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298540856 00000100:00000001:1.0:1787695969.145292:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298529473 00000100:00000001:1.0:1787695969.145294:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298540856 00000100:00000001:1.0:1787695969.145295:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695969.145297:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.145298:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695969.145300:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695969/real 1787695969] req@ffff8b14466f5f80 x1874531139554432/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695969.145305:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695969.145305:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695969.145306:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695969.145308:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695969.145309:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695969.145310:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695969.145312:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5f80 x1874531139554432/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695969.145316:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5f80 x1874531139554432/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.145321:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695969.145331:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.145333:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695969.145334:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695969.145342:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5f80 x1874531139554432/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.145349:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695969.145354:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5f80 x1874531139554432/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.145384:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695969.145387:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.145389:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695969.145391:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.145394:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695969.145395:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695969.145398:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695969.145402:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695969.145404:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695969.145408:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5f80 x1874531139554432/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695969.145414:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139554432:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695969.145417:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.145418:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695969.145420:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4700 x1874531139554176/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.145425:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695969.145426:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695969.145427:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695969.145429:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695969.145430:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695969.145431:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695969.145432:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695969.145434:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:1.0:1787695969.145436:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695969.145437:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695969.145438:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4700. 00000100:00000001:1.0:1787695969.145440:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695969.145441:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.145446:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695969.145448:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5500 x1874531139554304/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.145453:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695969.145453:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695969.145454:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695969.145456:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695969.145457:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695969.145457:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695969.145458:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695969.145459:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676000. 02000000:00000001:1.0:1787695969.145461:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695969.145462:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695969.145462:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5500. 00000100:00000001:1.0:1787695969.145464:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695969.145465:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.145466:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695969.145468:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5f80 x1874531139554432/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695994 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695969.145472:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695969.145472:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695969.145473:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695969.145474:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695969.145475:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695969.145475:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695969.145477:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695969.145477:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:1.0:1787695969.145479:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695969.145480:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695969.145480:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5f80. 00000100:00000001:1.0:1787695969.145482:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695969.145482:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.145483:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695969.145486:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695969.145487:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695969.145488:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695969.145493:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695969.206481:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695969.206489:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695969.206492:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695969.206497:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695969.206500:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695969.206504:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695969.206507:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695969.206520:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695969.206529:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695969.846430:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695969.846435:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695969.846439:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695969.846440:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695969.846443:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695969.846444:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695969.846453:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695969.846453:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695969.846456:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695969.846457:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695969.846459:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695969.846460:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695969.846514:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695969.846518:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695969.846521:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695970.230511:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695970.230521:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695970.230524:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695970.230531:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695970.230536:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695970.230540:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695970.230544:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695970.230548:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695970.230551:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695970.870496:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695970.870503:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695970.870509:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695970.870511:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695970.870515:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695970.870516:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695970.870520:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695970.870522:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695970.870525:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695970.870526:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695970.870529:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695970.870531:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695970.870604:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695970.870607:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695970.870609:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695971.254460:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695971.254466:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695971.254468:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695971.254472:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695971.254475:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695971.254478:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695971.254480:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695971.254482:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695971.254484:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695971.894465:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695971.894470:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695971.894475:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695971.894476:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695971.894479:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695971.894479:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695971.894482:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695971.894483:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695971.894485:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695971.894488:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695971.894490:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695971.894494:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695971.894495:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695971.894496:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695971.894514:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695971.894515:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695971.894517:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695971.894519:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695971.894521:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695971.894522:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695971.894579:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695971.894583:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695971.894586:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695972.278471:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695972.278479:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695972.278482:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695972.278487:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695972.278491:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695972.278495:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695972.278515:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695972.278518:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695972.278521:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695972.918465:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695972.918470:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695972.918475:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695972.918476:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695972.918478:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695972.918479:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695972.918481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695972.918482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695972.918485:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695972.918486:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695972.918488:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695972.918488:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695972.918554:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695972.918558:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695972.918560:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695973.302479:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695973.302485:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695973.302487:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695973.302491:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695973.302494:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695973.302497:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695973.302499:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695973.302501:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695973.302503:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695973.942489:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695973.942496:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695973.942498:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695973.942500:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695973.942509:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695973.942511:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695973.942512:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695973.942514:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695973.942517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695973.942518:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695973.942519:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695973.942520:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695973.942521:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695973.942522:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695973.942523:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695973.942525:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695973.942527:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695973.942528:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695973.942531:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695973.942532:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695973.942553:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695973.942554:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695973.942556:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695973.942557:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695973.942615:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695973.942618:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695973.942621:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787695974.262452:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695974.262463:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787695974.262466:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695974.262470:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695974.262475:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695974.262483:0:47074:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3861 00000400:00000200:3.0:1787695974.262494:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695974.262501:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695974.262505:0:47074:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695974.262509:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.262513:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695974.262516:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.262519:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695974.262523:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695974.262526:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:3.0:1787695974.262530:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695974.262533:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787695974.262536:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787695974.262539:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787695974.262558:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.262561:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695974.262563:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.262566:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695974.262569:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787695974.262573:0:47074:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695974.262576:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695974.262579:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695974.262583:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695974.262586:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695974.262588:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787695974.262591:0:47074:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695974.262597:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6d80. 00000020:00000040:3.0:1787695974.262601:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787695974.262605:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695974.262607:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695974.262610:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695974.262613:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695974.262619:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695974.262627:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674800. 00000100:00000001:3.0:1787695974.262635:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695974.262646:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6d80 x1874531139554560/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695974.262658:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695974.262663:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695974.262668:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009c3be57b] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6d80 x1874531139554560/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695974.262692:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695974.262696:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695974.262700:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695974.262701:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695974.262711:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695974.262720:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695974.262723:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3861 00000400:00000200:3.0:1787695974.262728:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695974.262732:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695974.262735:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695974.262738:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.262741:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695974.262744:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.262746:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695974.262748:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695974.262750:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:3.0:1787695974.262752:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695974.262754:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:3.0:1787695974.262756:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:3.0:1787695974.262758:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:3.0:1787695974.262777:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.262779:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695974.262783:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.262785:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695974.262787:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:3.0:1787695974.262791:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695974.262794:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695974.262796:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695974.262801:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695974.262803:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695974.262806:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695974.262810:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695974.262814:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695974.262816:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695974.262826:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5180. 00000020:00000040:3.0:1787695974.262828:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695974.262831:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695974.262832:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695974.262834:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695974.262836:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695974.262838:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695974.262843:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 00000100:00000001:3.0:1787695974.262846:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695974.262852:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5180 x1874531139554688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695974.262859:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695974.262862:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695974.262865:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a115abf4] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5180 x1874531139554688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695974.262885:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695974.262887:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695974.262891:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695974.262892:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695974.262894:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695974.262896:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695974.262899:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3861 00000400:00000200:3.0:1787695974.262904:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695974.262907:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695974.262909:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695974.262911:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.262914:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695974.262916:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.262917:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695974.262919:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695974.262926:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:3.0:1787695974.262928:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695974.262930:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:3.0:1787695974.262932:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:3.0:1787695974.262935:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:3.0:1787695974.262938:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.262940:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695974.262942:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.262944:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695974.262946:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:3.0:1787695974.262949:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695974.262951:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695974.262953:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695974.262957:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695974.262959:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695974.262961:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695974.262963:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695974.262966:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695974.262968:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695974.262971:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4380. 00000020:00000040:3.0:1787695974.262973:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695974.262976:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695974.262977:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695974.262979:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695974.262980:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695974.262982:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695974.262987:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676000. 00000100:00000001:3.0:1787695974.262990:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695974.262995:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4380 x1874531139554816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695974.263003:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695974.263006:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695974.263020:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1787695974.263020:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d2982c15] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4380 x1874531139554816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.263027:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695974.263032:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787695974.263035:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695974.263040:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1787695974.263044:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6d80 x1874531139554560/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695974.263045:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787695974.263049:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695974.263052:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695974.263059:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695974.263065:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3861 00000100:00000001:1.0:1787695974.263066:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695974.263071:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1787695974.263072:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:1.0:1787695974.263076:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000200:3.0:1787695974.263078:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 02000000:00000001:1.0:1787695974.263081:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1787695974.263083:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695974.263089:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.263091:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695974.263093:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.263095:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695974.263098:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695974.263099:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 02000000:00000001:1.0:1787695974.263102:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695974.263102:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695974.263105:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:3.0:1787695974.263110:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00100000:1.0:1787695974.263111:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139554560:192.168.201.140@tcp:250:kworker.0 00000020:00000001:3.0:1787695974.263116:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:3.0:1787695974.263122:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.263124:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695974.263126:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.263129:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695974.263130:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1787695974.263132:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 02000000:00000001:1.0:1787695974.263137:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1787695974.263137:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000001:1.0:1787695974.263142:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695974.263143:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695974.263148:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787695974.263152:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:1.0:1787695974.263156:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:3.0:1787695974.263157:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695974.263162:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695974.263163:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695974.263167:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695974.263170:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000400:00000200:1.0:1787695974.263173:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000008:00000001:3.0:1787695974.263175:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695974.263179:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695974.263181:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000100:00000010:3.0:1787695974.263184:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5f80. 00000020:00000040:3.0:1787695974.263189:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000400:00000010:1.0:1787695974.263191:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 02000000:00000001:3.0:1787695974.263192:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695974.263196:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695974.263198:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695974.263200:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:1.0:1787695974.263202:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139554560, portal 25 02000000:00000001:3.0:1787695974.263203:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695974.263209:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000010:3.0:1787695974.263212:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 00000100:00000001:1.0:1787695974.263213:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930155392 : -128555779396224 : ffff8b14466f6d80) 00000100:00000001:3.0:1787695974.263219:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695974.263225:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6d80 x1874531139554560/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695974.263228:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5f80 x1874531139554944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695974.263250:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787695974.263252:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695974.263256:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695974.263258:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695974.263263:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006e7bde76] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5f80 x1874531139554944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1787695974.263265:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695974.263273:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139554560, offset 0 00000100:00000001:3.0:1787695974.263277:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695974.263282:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695974.263285:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00080000:3.0:1787695974.263287:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1787695974.263289:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695974.263291:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695974.263297:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695974.263301:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3861 00000400:00000200:1.0:1787695974.263305:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695974.263310:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695974.263316:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:3.0:1787695974.263318:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695974.263324:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695974.263336:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.263338:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695974.263340:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1787695974.263342:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.263346:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695974.263348:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000020:00000001:3.0:1787695974.263350:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695974.263354:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000400:00000200:1.0:1787695974.263356:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000020:00000001:3.0:1787695974.263357:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695974.263362:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000400:00000200:1.0:1787695974.263393:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695974.263402:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17142:17142:256:4294967295] 192.168.201.140@tcp LPNI seq info [17141:17141:8:4294967295] 00000020:00000001:3.0:1787695974.263402:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:3.0:1787695974.263409:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000400:00000200:1.0:1787695974.263414:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695974.263417:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.263421:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695974.263425:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.263430:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695974.263435:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000020:00000040:3.0:1787695974.263436:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:3.0:1787695974.263444:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000800:00000200:1.0:1787695974.263448:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695974.263449:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695974.263454:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000010:1.0:1787695974.263457:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 02000000:00000001:3.0:1787695974.263458:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695974.263462:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695974.263465:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000200:1.0:1787695974.263469:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000008:00100000:3.0:1787695974.263469:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000800:00000200:1.0:1787695974.263476:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000008:00000001:3.0:1787695974.263476:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695974.263478:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695974.263486:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5500. 00000800:00000200:1.1:1787695974.263487:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000020:00000040:3.0:1787695974.263490:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:3.0:1787695974.263493:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695974.263496:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695974.263501:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.1:1787695974.263501:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 02000000:00000001:3.0:1787695974.263504:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695974.263508:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787695974.263513:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 02000000:00000010:3.0:1787695974.263516:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 00000800:00000010:1.0:1787695974.263517:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695974.263522:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1787695974.263530:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695974.263538:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5500 x1874531139555072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695974.263543:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695974.263554:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000100:00000001:3.0:1787695974.263555:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000010:1.0:1787695974.263559:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:3.0:1787695974.263559:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695974.263567:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1787695974.263568:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a777d68] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5500 x1874531139555072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695974.263575:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6d80 x1874531139554560/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695974.263581:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695974.263587:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695974.263593:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695974.263595:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695974.263596:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695974.263599:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695974.263602:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1787695974.263603:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3861 00000100:00000040:1.0:1787695974.263611:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6d80 x1874531139554560/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787695999 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787695974.263612:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695974.263619:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695974.263625:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695974.263627:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695974.263631:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:3.0:1787695974.263631:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.263635:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695974.263637:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695974.263640:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695974.263640:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695974.263646:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695974.263649:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000400:00000200:1.0:1787695974.263650:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:3.0:1787695974.263654:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695974.263656:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298545977 00000020:00000040:3.0:1787695974.263659:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000020:00000001:3.0:1787695974.263663:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:1.0:1787695974.263666:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298540856 00000020:00000001:3.0:1787695974.263666:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000040:3.0:1787695974.263671:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.263674:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695974.263679:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298545977 00000100:00000040:3.0:1787695974.263686:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695974.263687:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298540856 00000100:00000001:3.0:1787695974.263691:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695974.263693:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1787695974.263698:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00000001:1.0:1787695974.263699:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695974.263704:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1787695974.263706:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695974.263711:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695974.263713:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695974/real 1787695974] req@ffff8b14466f6d80 x1874531139554560/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695974.263715:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695974.263721:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695974.263726:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695974.263730:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000001:1.0:1787695974.263731:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00100000:3.0:1787695974.263733:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000001:1.0:1787695974.263735:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00000001:3.0:1787695974.263738:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695974.263740:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000100:00000001:3.0:1787695974.263743:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695974.263745:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000100:00000010:3.0:1787695974.263748:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4700. 00000400:00000010:1.0:1787695974.263750:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000020:00000040:3.0:1787695974.263754:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000100:00000001:1.0:1787695974.263756:0:2402:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1787695974.263761:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000200:1.0:1787695974.263764:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6d80 x1874531139554560/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787695974.263765:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695974.263771:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695974.263774:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695974.263777:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695974.263783:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6d80 x1874531139554560/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1787695974.263785:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675800. 00000100:00000001:3.0:1787695974.263792:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695974.263795:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695974.263799:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787695974.263802:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4700 x1874531139555200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695974.263803:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695974.263807:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695974.263817:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6d80 x1874531139554560/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695974.263818:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695974.263823:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695974.263832:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008818c2cc] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4700 x1874531139555200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.263835:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695974.263846:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6d80 x1874531139554560/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695974.263848:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695974.263853:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695974.263861:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787695974.263864:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787695974.263879:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695974.263883:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.263887:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:3.0:1787695974.263887:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695974.263892:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695974.263897:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3861 00000100:00000001:1.0:1787695974.263898:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.263903:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000400:00000200:3.0:1787695974.263905:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695974.263907:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:3.0:1787695974.263912:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695974.263914:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787695974.263920:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:3.0:1787695974.263920:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.263925:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:1.0:1787695974.263930:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000040:3.0:1787695974.263931:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695974.263937:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787695974.263937:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695974.263941:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695974.263943:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000100:00000040:1.0:1787695974.263946:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6d80 x1874531139554560/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695974.263946:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695974.263950:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:3.0:1787695974.263953:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000020:00000001:3.0:1787695974.263958:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:3.0:1787695974.263964:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00100000:1.0:1787695974.263967:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139554560:192.168.201.140@tcp:250:kworker.0 00000100:00000001:3.0:1787695974.263968:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695974.263973:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695974.263975:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695974.263976:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1787695974.263977:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00080000:3.0:1787695974.263983:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000040:1.0:1787695974.263985:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5180 x1874531139554688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695974.263985:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695974.263989:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695974.263993:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695974.263999:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695974.264001:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695974.264003:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695974.264005:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787695974.264007:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787695974.264010:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:3.0:1787695974.264012:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695974.264014:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695974.264016:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695974.264019:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695974.264023:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139554688:192.168.201.140@tcp:38:kworker.0 00000100:00000010:3.0:1787695974.264028:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4000. 00000100:00000001:1.0:1787695974.264030:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000040:3.0:1787695974.264033:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:1.0:1787695974.264035:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695974.264038:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695974.264039:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695974.264040:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787695974.264044:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695974.264046:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 00000100:00000001:3.0:1787695974.264048:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695974.264049:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695974.264053:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1787695974.264056:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787695974.264057:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695974.264060:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 02000000:00000010:3.0:1787695974.264063:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 00000400:00000010:1.0:1787695974.264066:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000001:3.0:1787695974.264067:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695974.264070:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139554688, portal 10 00000100:00000001:1.0:1787695974.264075:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695974.264076:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930148224 : -128555779403392 : ffff8b14466f5180) 00000100:00100000:3.0:1787695974.264081:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4000 x1874531139555328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1787695974.264090:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5180 x1874531139554688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695974.264095:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695974.264100:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787695974.264101:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695974.264104:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695974.264107:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695974.264110:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139554688, offset 0 00000100:00000040:3.0:1787695974.264111:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5a8f356] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4000 x1874531139555328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695974.264116:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695974.264122:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695974.264127:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695974.264136:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1787695974.264137:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695974.264141:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000040:3.0:1787695974.264143:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695974.264148:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000040:3.0:1787695974.264149:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695974.264155:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695974.264157:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000040:3.0:1787695974.264159:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1787695974.264164:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17143:17143:256:4294967295] 192.168.201.140@tcp LPNI seq info [17142:17142:8:4294967295] 00000100:00000040:3.0:1787695974.264166:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3872) 00000400:00000200:1.0:1787695974.264173:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695974.264181:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695974.264186:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695974.264190:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695974.264195:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695974.264197:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1787695974.264202:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695974.264205:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1787695974.264209:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695974.264210:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695974.264214:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695974.264218:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695974.264221:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695974.264223:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695974.264225:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695974.264229:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5180 x1874531139554688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264237:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695974.264246:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5180 x1874531139554688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695999 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264253:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695974.264254:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695974.264258:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695974.264262:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000400:00000200:1.0:1787695974.264266:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695974.264268:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298535878 00000100:00000001:1.0:1787695974.264271:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298545977 00000100:00000001:1.0:1787695974.264273:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298535878 00000100:00000001:1.0:1787695974.264275:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298545977 00000100:00000001:1.0:1787695974.264277:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695974.264279:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695974.264281:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695974.264284:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695974/real 1787695974] req@ffff8b14466f5180 x1874531139554688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695974.264292:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695974.264293:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695974.264295:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695974.264297:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695974.264299:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695974.264301:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695974.264304:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5180 x1874531139554688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695974.264312:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5180 x1874531139554688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264319:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695974.264320:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.264322:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695974.264323:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695974.264333:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5180 x1874531139554688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264340:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695974.264344:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5180 x1874531139554688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264351:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695974.264353:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.264354:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695974.264356:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.264358:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695974.264359:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695974.264362:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695974.264387:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695974.264390:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695974.264394:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5180 x1874531139554688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695974.264402:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139554688:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695974.264406:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.264409:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695974.264412:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6d80 x1874531139554560/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264419:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695974.264421:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695974.264424:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695974.264429:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695974.264431:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695974.264432:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:1.0:1787695974.264434:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695974.264442:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674800. 02000000:00000001:1.0:1787695974.264446:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695974.264448:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695974.264449:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6d80. 00000100:00000001:1.0:1787695974.264452:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695974.264453:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.264454:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695974.264458:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5180 x1874531139554688/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264462:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695974.264463:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695974.264465:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695974.264468:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695974.264470:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695974.264471:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695974.264473:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695974.264474:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:1.0:1787695974.264478:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695974.264479:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695974.264480:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5180. 00000100:00000001:1.0:1787695974.264481:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695974.264482:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.264484:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.264490:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695974.264491:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695974.264492:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695974.264494:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695974.264495:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695974.264498:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4380 x1874531139554816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264502:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695974.264503:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695974.264505:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695974.264506:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695974.264508:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695974.264509:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139554816:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695974.264515:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695974.264517:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695974.264518:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695974.264519:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695974.264521:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695974.264522:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695974.264525:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695974.264527:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695974.264529:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695974.264532:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139554816, portal 10 00000100:00000001:1.0:1787695974.264533:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695974.264534:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930144640 : -128555779406976 : ffff8b14466f4380) 00000100:00000040:1.0:1787695974.264538:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4380 x1874531139554816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264543:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695974.264544:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695974.264546:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695974.264547:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139554816, offset 0 00000400:00000200:1.0:1787695974.264550:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695974.264553:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695974.264555:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695974.264557:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695974.264559:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695974.264561:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695974.264563:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695974.264565:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17144:17144:256:4294967295] 192.168.201.140@tcp LPNI seq info [17143:17143:8:4294967295] 00000400:00000200:1.0:1787695974.264568:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695974.264572:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695974.264575:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695974.264577:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695974.264580:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695974.264585:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1787695974.264588:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695974.264590:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1787695974.264592:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695974.264594:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695974.264596:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695974.264599:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695974.264601:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695974.264603:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695974.264604:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695974.264607:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4380 x1874531139554816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264612:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695974.264614:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4380 x1874531139554816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695999 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264619:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695974.264620:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695974.264622:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695974.264624:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787695974.264626:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695974.264628:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298534727 00000100:00000001:1.0:1787695974.264629:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298545977 00000100:00000001:1.0:1787695974.264631:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298534727 00000100:00000001:1.0:1787695974.264632:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298545977 00000100:00000001:1.0:1787695974.264634:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695974.264635:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695974.264637:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695974.264639:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695974/real 1787695974] req@ffff8b14466f4380 x1874531139554816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695974.264644:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695974.264644:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695974.264649:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695974.264650:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695974.264651:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695974.264653:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695974.264656:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4380 x1874531139554816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695974.264661:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4380 x1874531139554816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264666:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695974.264667:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.264668:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695974.264669:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695974.264671:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4380 x1874531139554816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264675:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695974.264677:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4380 x1874531139554816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264681:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695974.264683:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.264684:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695974.264685:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.264686:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695974.264687:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695974.264688:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695974.264690:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695974.264692:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695974.264694:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4380 x1874531139554816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695974.264699:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139554816:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695974.264706:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695974.264709:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5f80 x1874531139554944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264713:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695974.264714:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695974.264716:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695974.264717:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695974.264718:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695974.264720:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139554944:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695974.264722:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695974.264724:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695974.264725:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695974.264726:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695974.264728:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695974.264729:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695974.264732:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695974.264733:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695974.264736:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695974.264738:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139554944, portal 4 00000100:00000001:1.0:1787695974.264739:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695974.264740:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930151808 : -128555779399808 : ffff8b14466f5f80) 00000100:00000040:1.0:1787695974.264744:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5f80 x1874531139554944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264751:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695974.264753:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695974.264756:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695974.264758:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139554944, offset 0 00000400:00000200:1.0:1787695974.264763:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695974.264767:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695974.264770:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695974.264773:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695974.264782:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695974.264784:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695974.264787:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695974.264790:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17145:17145:256:4294967295] 192.168.201.140@tcp LPNI seq info [17144:17144:8:4294967295] 00000400:00000200:1.0:1787695974.264794:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695974.264801:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695974.264806:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695974.264810:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695974.264814:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695974.264816:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1787695974.264819:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695974.264821:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1787695974.264823:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695974.264825:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695974.264828:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695974.264830:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695974.264832:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695974.264833:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695974.264835:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695974.264838:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5f80 x1874531139554944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264844:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695974.264846:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5f80 x1874531139554944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695999 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264850:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695974.264851:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695974.264854:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695974.264857:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695974.264860:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695974.264862:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298534757 00000100:00000001:1.0:1787695974.264865:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298545977 00000100:00000001:1.0:1787695974.264881:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298534757 00000100:00000001:1.0:1787695974.264888:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298545977 00000100:00000001:1.0:1787695974.264891:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695974.264893:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695974.264895:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695974.264900:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695974/real 1787695974] req@ffff8b14466f5f80 x1874531139554944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695974.264909:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695974.264911:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695974.264913:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695974.264915:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695974.264916:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695974.264918:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695974.264920:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5f80 x1874531139554944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695974.264926:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5f80 x1874531139554944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264930:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695974.264931:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.264933:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695974.264933:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695974.264936:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5f80 x1874531139554944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264940:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695974.264942:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5f80 x1874531139554944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264947:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695974.264948:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.264949:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695974.264950:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.264955:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695974.264956:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695974.264959:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695974.264961:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695974.264963:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695974.264965:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5f80 x1874531139554944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695974.264973:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139554944:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695974.264977:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695974.264980:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5500 x1874531139555072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.264987:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695974.264988:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695974.264990:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695974.264993:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695974.264995:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695974.264997:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139555072:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695974.265001:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695974.265002:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695974.265004:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695974.265006:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695974.265009:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695974.265011:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695974.265015:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695974.265018:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695974.265022:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695974.265025:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139555072, portal 4 00000100:00000001:1.0:1787695974.265027:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695974.265028:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930149120 : -128555779402496 : ffff8b14466f5500) 00000100:00000040:1.0:1787695974.265033:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5500 x1874531139555072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265042:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695974.265044:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695974.265045:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695974.265047:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139555072, offset 0 00000400:00000200:1.0:1787695974.265050:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695974.265058:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695974.265061:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695974.265063:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695974.265065:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695974.265067:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695974.265068:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695974.265071:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17146:17146:256:4294967295] 192.168.201.140@tcp LPNI seq info [17145:17145:8:4294967295] 00000400:00000200:1.0:1787695974.265073:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695974.265077:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695974.265080:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695974.265083:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695974.265085:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695974.265087:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1787695974.265090:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695974.265092:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1787695974.265094:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695974.265096:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695974.265098:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695974.265100:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695974.265102:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695974.265103:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695974.265105:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695974.265108:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5500 x1874531139555072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265113:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695974.265115:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5500 x1874531139555072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787695999 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265127:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695974.265128:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695974.265131:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695974.265133:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695974.265135:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695974.265136:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298534429 00000100:00000001:1.0:1787695974.265138:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298545977 00000100:00000001:1.0:1787695974.265140:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298534429 00000100:00000001:1.0:1787695974.265141:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298545977 00000100:00000001:1.0:1787695974.265142:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695974.265144:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695974.265145:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695974.265147:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695974/real 1787695974] req@ffff8b14466f5500 x1874531139555072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695974.265152:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695974.265153:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695974.265155:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695974.265156:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695974.265157:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695974.265159:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695974.265161:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5500 x1874531139555072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695974.265166:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5500 x1874531139555072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265169:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695974.265170:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.265171:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695974.265172:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695974.265174:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5500 x1874531139555072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265182:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695974.265185:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5500 x1874531139555072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265189:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695974.265190:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.265191:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695974.265192:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.265193:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695974.265194:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695974.265196:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695974.265197:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695974.265199:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695974.265201:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5500 x1874531139555072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695974.265206:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139555072:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695974.265209:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695974.265211:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4700 x1874531139555200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265215:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695974.265216:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695974.265217:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695974.265218:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695974.265219:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695974.265221:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139555200:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695974.265223:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695974.265224:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695974.265229:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695974.265230:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695974.265232:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695974.265234:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695974.265236:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695974.265238:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695974.265240:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695974.265242:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139555200, portal 10 00000100:00000001:1.0:1787695974.265243:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695974.265244:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930145536 : -128555779406080 : ffff8b14466f4700) 00000100:00000040:1.0:1787695974.265249:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4700 x1874531139555200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265253:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695974.265255:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695974.265256:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695974.265258:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139555200, offset 0 00000400:00000200:1.0:1787695974.265260:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695974.265262:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695974.265264:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695974.265266:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695974.265268:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695974.265269:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695974.265271:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695974.265273:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17147:17147:256:4294967295] 192.168.201.140@tcp LPNI seq info [17146:17146:8:4294967295] 00000400:00000200:1.0:1787695974.265276:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695974.265280:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695974.265282:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695974.265284:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695974.265286:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695974.265289:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1787695974.265291:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695974.265293:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1787695974.265295:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695974.265300:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695974.265302:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695974.265304:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695974.265306:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695974.265307:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695974.265309:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695974.265311:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4700 x1874531139555200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265317:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695974.265318:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4700 x1874531139555200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695999 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265323:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695974.265324:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695974.265325:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695974.265328:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695974.265329:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695974.265331:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298534615 00000100:00000001:1.0:1787695974.265333:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298545977 00000100:00000001:1.0:1787695974.265335:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298534615 00000100:00000001:1.0:1787695974.265338:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298545977 00000100:00000001:1.0:1787695974.265340:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695974.265342:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695974.265344:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695974.265348:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695974/real 1787695974] req@ffff8b14466f4700 x1874531139555200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695974.265355:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695974.265357:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695974.265358:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695974.265361:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695974.265362:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695974.265399:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695974.265402:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4700 x1874531139555200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695974.265409:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4700 x1874531139555200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265414:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695974.265415:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.265417:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695974.265418:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695974.265421:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4700 x1874531139555200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265425:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695974.265428:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4700 x1874531139555200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265432:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695974.265433:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.265434:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695974.265435:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.265437:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695974.265438:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695974.265440:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695974.265442:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695974.265443:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695974.265445:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4700 x1874531139555200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695974.265450:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139555200:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695974.265452:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695974.265458:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4000 x1874531139555328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265462:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695974.265463:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695974.265465:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695974.265466:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695974.265467:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695974.265469:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139555328:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695974.265471:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695974.265472:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695974.265473:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695974.265475:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695974.265478:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695974.265479:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695974.265481:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695974.265483:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695974.265486:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695974.265488:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139555328, portal 10 00000100:00000001:1.0:1787695974.265489:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695974.265490:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930143744 : -128555779407872 : ffff8b14466f4000) 00000100:00000040:1.0:1787695974.265494:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4000 x1874531139555328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265498:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695974.265500:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695974.265501:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695974.265503:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139555328, offset 0 00000400:00000200:1.0:1787695974.265505:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695974.265508:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695974.265510:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695974.265512:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695974.265514:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695974.265515:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695974.265520:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695974.265523:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17148:17148:256:4294967295] 192.168.201.140@tcp LPNI seq info [17147:17147:8:4294967295] 00000400:00000200:1.0:1787695974.265525:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695974.265530:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695974.265533:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695974.265535:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695974.265538:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695974.265540:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000200:1.1:1787695974.265542:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695974.265544:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 27 secs later) 00000800:00000100:1.0:1787695974.265547:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695974.265548:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695974.265550:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695974.265552:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695974.265554:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695974.265555:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695974.265557:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695974.265559:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4000 x1874531139555328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695999 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265564:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695974.265566:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4000 x1874531139555328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787695999 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265570:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695974.265571:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695974.265573:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695974.265575:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695974.265577:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695974.265578:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298534594 00000100:00000001:1.0:1787695974.265580:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298545977 00000100:00000001:1.0:1787695974.265581:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298534594 00000100:00000001:1.0:1787695974.265583:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298545977 00000100:00000001:1.0:1787695974.265587:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695974.265588:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695974.265589:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695974.265592:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695974/real 1787695974] req@ffff8b14466f4000 x1874531139555328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695974.265598:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695974.265599:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695974.265600:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695974.265601:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695974.265602:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695974.265603:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695974.265605:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4000 x1874531139555328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695974.265610:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4000 x1874531139555328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265614:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695974.265615:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.265616:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695974.265617:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695974.265619:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4000 x1874531139555328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265623:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695974.265626:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4000 x1874531139555328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265630:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695974.265631:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.265632:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695974.265633:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.265634:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695974.265635:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695974.265639:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695974.265641:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695974.265643:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695974.265646:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4000 x1874531139555328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695974.265650:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139555328:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695974.265653:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.265654:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695974.265656:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4380 x1874531139554816/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265660:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695974.265661:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695974.265662:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695974.265664:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695974.265664:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695974.265666:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695974.265667:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695974.265669:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676000. 02000000:00000001:1.0:1787695974.265672:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695974.265673:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695974.265674:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4380. 00000100:00000001:1.0:1787695974.265675:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695974.265676:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.265677:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695974.265679:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5f80 x1874531139554944/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265683:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695974.265684:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695974.265685:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695974.265686:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695974.265687:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695974.265691:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695974.265693:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695974.265694:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:1.0:1787695974.265696:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695974.265696:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695974.265697:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5f80. 00000100:00000001:1.0:1787695974.265699:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695974.265700:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.265701:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695974.265703:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5500 x1874531139555072/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265707:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695974.265707:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695974.265708:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695974.265710:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695974.265711:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695974.265711:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695974.265713:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695974.265714:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:1.0:1787695974.265715:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695974.265716:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695974.265717:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5500. 00000100:00000001:1.0:1787695974.265718:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695974.265719:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.265720:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695974.265722:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4700 x1874531139555200/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265725:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695974.265726:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695974.265727:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695974.265728:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695974.265729:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695974.265729:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695974.265730:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695974.265731:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675800. 02000000:00000001:1.0:1787695974.265733:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695974.265737:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695974.265739:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4700. 00000100:00000001:1.0:1787695974.265741:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695974.265742:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.265744:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695974.265748:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4000 x1874531139555328/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787695999 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695974.265755:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695974.265756:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695974.265758:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695974.265762:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695974.265763:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695974.265765:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695974.265767:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695974.265769:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:1.0:1787695974.265771:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695974.265773:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695974.265774:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4000. 00000100:00000001:1.0:1787695974.265776:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695974.265778:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.265780:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695974.265784:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695974.265786:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695974.265787:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695974.265789:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695974.326453:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695974.326458:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695974.326460:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695974.326463:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695974.326466:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695974.326469:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695974.326471:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695974.326473:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695974.326475:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695974.966460:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695974.966467:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695974.966473:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695974.966475:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695974.966479:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695974.966480:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695974.966484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695974.966485:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695974.966488:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695974.966490:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695974.966493:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695974.966494:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695974.966564:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695974.966569:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695974.966572:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695975.350456:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695975.350478:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695975.350480:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695975.350484:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695975.350488:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695975.350491:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695975.350494:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695975.350497:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695975.350499:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695975.990476:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695975.990482:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695975.990487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695975.990488:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695975.990490:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695975.990491:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695975.990494:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695975.990495:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695975.990499:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695975.990501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695975.990505:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695975.990506:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695975.990559:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695975.990563:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695975.990566:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695976.374471:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695976.374477:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695976.374479:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695976.374483:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695976.374485:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695976.374488:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695976.374490:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695976.374491:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695976.374494:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695977.014439:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695977.014446:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695977.014451:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695977.014452:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695977.014456:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695977.014459:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695977.014462:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695977.014469:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695977.014471:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695977.014473:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695977.014476:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695977.014478:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695977.014480:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695977.014492:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695977.014496:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695977.014498:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695977.014500:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695977.014502:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695977.014503:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695977.014504:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695977.014506:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695977.014507:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695977.014509:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695977.014512:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695977.014515:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695977.014516:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695977.014520:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695977.014539:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695977.014540:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695977.014542:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695977.014544:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695977.014545:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695977.014548:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695977.014549:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695977.014551:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695977.014553:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695977.014612:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695977.014627:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695977.014630:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695977.398497:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695977.398506:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695977.398508:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695977.398513:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695977.398517:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695977.398521:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695977.398523:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695977.398525:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695977.398528:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695978.038515:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695978.038521:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695978.038525:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695978.038526:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695978.038529:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695978.038530:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695978.038532:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695978.038533:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695978.038536:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695978.038537:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695978.038539:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695978.038540:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695978.038598:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695978.038602:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695978.038605:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695978.422440:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695978.422447:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695978.422448:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695978.422452:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695978.422456:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695978.422458:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695978.422460:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695978.422462:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695978.422464:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695979.062436:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695979.062444:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695979.062451:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695979.062452:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695979.062456:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695979.062458:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695979.062462:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695979.062463:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695979.062465:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695979.062466:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695979.062469:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695979.062470:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695979.062523:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695979.062527:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695979.062530:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787695979.382993:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695979.383004:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787695979.383007:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695979.383010:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695979.383016:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695979.383023:0:47074:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3867 00000400:00000200:2.0:1787695979.383033:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695979.383039:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695979.383043:0:47074:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695979.383047:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695979.383051:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695979.383054:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695979.383056:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695979.383060:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695979.383062:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:2.0:1787695979.383065:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695979.383068:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:2.0:1787695979.383081:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:2.0:1787695979.383083:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:2.0:1787695979.383087:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695979.383089:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695979.383091:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695979.383093:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695979.383097:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:2.0:1787695979.383100:0:47074:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695979.383102:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695979.383105:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695979.383109:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695979.383112:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695979.383114:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787695979.383117:0:47074:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695979.383123:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ace00. 00000020:00000040:2.0:1787695979.383126:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:2.0:1787695979.383129:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695979.383131:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695979.383134:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695979.383137:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695979.383142:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695979.383149:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445662800. 00000100:00000001:2.0:1787695979.383156:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695979.383165:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ace00 x1874531139555456/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695979.383176:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695979.383180:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695979.383185:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000046eddc29] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ace00 x1874531139555456/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695979.383207:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695979.383211:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695979.383220:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787695979.383222:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695979.383224:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695979.383227:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695979.383230:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3867 00000400:00000200:2.0:1787695979.383234:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695979.383238:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695979.383240:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695979.383244:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695979.383246:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695979.383249:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695979.383251:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695979.383254:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787695979.383254:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695979.383257:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000100:00000001:1.0:1787695979.383261:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1787695979.383261:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695979.383264:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000100:00000001:1.0:1787695979.383266:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1787695979.383268:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:2.0:1787695979.383272:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:1.0:1787695979.383279:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ace00 x1874531139555456/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695979.383279:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695979.383284:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695979.383289:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695979.383294:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695979.383300:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:2.0:1787695979.383303:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695979.383305:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695979.383306:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695979.383312:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695979.383317:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695979.383322:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695979.383327:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:2.0:1787695979.383328:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787695979.383332:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695979.383338:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:2.0:1787695979.383340:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695979.383343:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695979.383344:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695979.383351:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695979.383352:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ae680. 00000020:00000040:2.0:1787695979.383357:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000100:00100000:1.0:1787695979.383359:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139555456:192.168.201.140@tcp:250:kworker.0 02000000:00000001:2.0:1787695979.383361:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695979.383400:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695979.383404:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787695979.383407:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695979.383408:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695979.383412:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695979.383413:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000010:2.0:1787695979.383418:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661c00. 02000000:00000001:1.0:1787695979.383419:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695979.383426:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787695979.383427:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695979.383433:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546000. 00000100:00100000:2.0:1787695979.383435:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ae680 x1874531139555584/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787695979.383438:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695979.383449:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695979.383449:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695979.383455:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695979.383458:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000100:00000040:2.0:1787695979.383463:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000019831621] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ae680 x1874531139555584/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1787695979.383468:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000001:2.0:1787695979.383479:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695979.383480:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139555456, portal 25 00000100:00080000:2.0:1787695979.383483:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695979.383485:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695979.383489:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928803840 : -128555780747776 : ffff8b14465ace00) 00000100:00080000:2.0:1787695979.383491:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787695979.383495:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695979.383497:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1787695979.383501:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ace00 x1874531139555456/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695979.383502:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695979.383508:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3867 00000400:00000200:2.0:1787695979.383519:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695979.383521:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:2.0:1787695979.383526:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000040:1.0:1787695979.383531:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00080000:2.0:1787695979.383532:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000400:00000010:1.0:1787695979.383538:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000040:2.0:1787695979.383540:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695979.383545:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1787695979.383546:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139555456, offset 0 00000100:00000040:2.0:1787695979.383550:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695979.383558:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695979.383564:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695979.383569:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:1.0:1787695979.383571:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000040:2.0:1787695979.383572:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000400:00000200:1.0:1787695979.383579:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000020:00000001:2.0:1787695979.383584:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:1.0:1787695979.383587:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:2.0:1787695979.383588:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:2.0:1787695979.383592:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:2.0:1787695979.383594:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000400:00000200:1.0:1787695979.383596:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000040:2.0:1787695979.383600:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695979.383603:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695979.383606:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000040:2.0:1787695979.383607:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695979.383612:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695979.383615:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000020:00000040:2.0:1787695979.383616:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:2.0:1787695979.383619:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695979.383621:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695979.383622:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695979.383624:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695979.383627:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695979.383629:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695979.383633:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1787695979.383634:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17149:17149:256:4294967295] 192.168.201.140@tcp LPNI seq info [17148:17148:8:4294967295] 00000008:00000001:2.0:1787695979.383635:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695979.383639:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695979.383645:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ac700. 00000400:00000200:1.0:1787695979.383649:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000040:2.0:1787695979.383650:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:2.0:1787695979.383656:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695979.383657:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695979.383658:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695979.383660:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695979.383662:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695979.383667:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445660000. 00000400:00000200:1.0:1787695979.383670:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:2.0:1787695979.383672:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695979.383680:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ac700 x1874531139555712/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1787695979.383684:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695979.383694:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000100:00000001:2.0:1787695979.383704:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:1.0:1787695979.383705:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:2.0:1787695979.383708:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695979.383712:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000100:00000040:2.0:1787695979.383714:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007c6208f8] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ac700 x1874531139555712/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1787695979.383725:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:2.0:1787695979.383726:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695979.383732:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.1:1787695979.383733:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1787695979.383739:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695979.383742:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000100:00080000:2.0:1787695979.383743:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787695979.383748:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:1.0:1787695979.383750:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:2.0:1787695979.383750:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695979.383756:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:1.0:1787695979.383762:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00080000:2.0:1787695979.383762:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3867 00000400:00000200:2.0:1787695979.383770:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695979.383773:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695979.383774:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695979.383776:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695979.383793:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695979.383795:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:1.0:1787695979.383797:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000100:00000001:2.0:1787695979.383797:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695979.383803:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000010:1.0:1787695979.383804:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000020:00000040:2.0:1787695979.383804:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:2.0:1787695979.383807:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695979.383811:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000100:00000001:1.0:1787695979.383813:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:2.0:1787695979.383813:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:2.0:1787695979.383818:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:2.0:1787695979.383820:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695979.383823:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1787695979.383825:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ace00 x1874531139555456/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695979.383826:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695979.383830:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695979.383839:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:2.0:1787695979.383843:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695979.383845:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787695979.383846:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695979.383849:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695979.383852:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1787695979.383854:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ace00 x1874531139555456/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696004 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695979.383855:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695979.383859:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695979.383862:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787695979.383867:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695979.383871:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.383876:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695979.383876:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465af480. 00000020:00000040:2.0:1787695979.383881:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000100:00000001:1.0:1787695979.383882:0:2402:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1787695979.383885:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695979.383887:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695979.383888:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695979.383890:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787695979.383891:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695979.383892:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695979.383899:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445662400. 00000400:00000200:1.0:1787695979.383903:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695979.383905:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.383909:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298551097 00000100:00100000:2.0:1787695979.383913:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465af480 x1874531139555840/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695979.383919:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298545977 00000100:00000001:2.0:1787695979.383920:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695979.383924:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.383927:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298551097 00000100:00000040:2.0:1787695979.383929:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000071cfa937] to pc [ptlrpcd_rcv+-1] req@ffff8b14465af480 x1874531139555840/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.383935:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298545977 00000100:00000001:2.0:1787695979.383937:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695979.383942:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695979.383943:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787695979.383945:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1787695979.383949:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787695979.383950:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695979.383951:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695979.383953:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695979.383962:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3867 00000100:00000001:1.0:1787695979.383963:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000400:00000200:2.0:1787695979.383967:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695979.383973:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00100000:1.0:1787695979.383974:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695979/real 1787695979] req@ffff8b14465ace00 x1874531139555456/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695979.383975:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695979.383980:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695979.383985:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695979.383988:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695979.383993:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695979.383997:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695979.383999:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:2.0:1787695979.384001:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695979.384003:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000400:00000001:1.0:1787695979.384004:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1787695979.384005:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000400:00000001:1.0:1787695979.384009:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1787695979.384009:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:2.0:1787695979.384014:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000010:1.0:1787695979.384015:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000100:00000001:2.0:1787695979.384016:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695979.384020:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695979.384022:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695979.384023:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000020:00000040:2.0:1787695979.384024:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000400:00000010:1.0:1787695979.384029:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00080000:2.0:1787695979.384030:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695979.384036:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.384038:0:2402:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1787695979.384038:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695979.384042:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695979.384045:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695979.384048:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ace00 x1874531139555456/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695979.384048:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695979.384051:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787695979.384056:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695979.384060:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695979.384065:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465af800. 00000020:00000040:2.0:1787695979.384069:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:2.0:1787695979.384073:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000200:1.0:1787695979.384074:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ace00 x1874531139555456/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695979.384079:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695979.384083:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695979.384086:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695979.384089:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695979.384095:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661800. 00000100:00000001:1.0:1787695979.384096:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695979.384098:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.384100:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.384106:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:2.0:1787695979.384106:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465af800 x1874531139555968/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695979.384110:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695979.384118:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695979.384120:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695979.384123:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b410802f] to pc [ptlrpcd_rcv+-1] req@ffff8b14465af800 x1874531139555968/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695979.384128:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695979.384130:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695979.384133:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1787695979.384135:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ace00 x1874531139555456/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695979.384135:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695979.384138:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695979.384140:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695979.384143:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3867 00000400:00000200:2.0:1787695979.384150:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695979.384156:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695979.384161:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695979.384165:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695979.384165:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695979.384171:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695979.384175:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695979.384179:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ace00 x1874531139555456/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695979.384180:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695979.384185:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695979.384188:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:2.0:1787695979.384192:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695979.384196:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000100:00000001:1.0:1787695979.384198:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1787695979.384202:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:1.0:1787695979.384204:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.384206:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695979.384208:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787695979.384214:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:1.0:1787695979.384215:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695979.384219:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:2.0:1787695979.384224:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00080000:1.0:1787695979.384226:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1787695979.384230:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:1.0:1787695979.384235:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000040:2.0:1787695979.384236:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695979.384238:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695979.384240:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:2.0:1787695979.384242:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695979.384246:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695979.384249:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695979.384250:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1787695979.384251:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695979.384256:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695979.384258:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695979.384260:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000040:1.0:1787695979.384262:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ace00 x1874531139555456/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1787695979.384263:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695979.384267:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695979.384272:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ad880. 00000020:00000040:2.0:1787695979.384277:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:2.0:1787695979.384281:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695979.384284:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695979.384287:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139555456:192.168.201.140@tcp:250:kworker.0 00000100:00000001:2.0:1787695979.384287:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695979.384289:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695979.384292:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695979.384297:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663c00. 00000100:00000001:1.0:1787695979.384299:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695979.384301:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.384303:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1787695979.384308:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ad880 x1874531139556096/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1787695979.384313:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ace00 x1874531139555456/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695979.384323:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695979.384327:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695979.384334:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000075b4d632] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ad880 x1874531139556096/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.384336:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695979.384340:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695979.384345:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546000. 00000100:00000001:2.0:1787695979.384345:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695979.384353:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:2.0:1787695979.384353:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:1.0:1787695979.384358:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000100:00080000:2.0:1787695979.384358:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787695979.384362:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695979.384386:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000020:00000040:1.0:1787695979.384389:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000100:00000001:2.0:1787695979.384390:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695979.384395:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3867 00000020:00000001:1.0:1787695979.384397:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000400:00000200:2.0:1787695979.384401:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000010:1.0:1787695979.384403:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445662800. 00000400:00000200:2.0:1787695979.384407:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695979.384410:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1787695979.384411:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:2.0:1787695979.384413:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695979.384415:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787695979.384416:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000010:1.0:1787695979.384420:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ace00. 00000100:00000040:2.0:1787695979.384421:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695979.384427:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695979.384429:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000020:00000001:2.0:1787695979.384430:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695979.384433:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1787695979.384434:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000100:00000001:1.0:1787695979.384439:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787695979.384440:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695979.384444:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:2.0:1787695979.384449:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:1.0:1787695979.384451:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:2.0:1787695979.384452:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:1.0:1787695979.384456:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695979.384460:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695979.384462:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695979.384465:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695979.384466:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695979.384470:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695979.384472:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695979.384478:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ae680 x1874531139555584/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695979.384479:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695979.384486:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00080000:2.0:1787695979.384495:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695979.384498:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695979.384500:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.384502:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695979.384504:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695979.384507:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695979.384510:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695979.384511:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695979.384514:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695979.384517:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787695979.384520:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695979.384527:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139555584:192.168.201.140@tcp:38:kworker.0 00000008:00100000:2.0:1787695979.384529:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787695979.384533:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:2.0:1787695979.384535:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695979.384538:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787695979.384539:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695979.384542:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695979.384544:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465aca80. 02000000:00000001:1.0:1787695979.384546:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:2.0:1787695979.384550:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000010:1.0:1787695979.384553:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546000. 02000000:00000001:2.0:1787695979.384556:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695979.384558:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695979.384560:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695979.384565:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1787695979.384566:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:2.0:1787695979.384568:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1787695979.384571:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 02000000:00000001:2.0:1787695979.384573:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695979.384578:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 02000000:00000010:2.0:1787695979.384581:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445662c00. 00000100:00000200:1.0:1787695979.384584:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139555584, portal 10 00000100:00000001:2.0:1787695979.384587:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.384589:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695979.384592:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928810112 : -128555780741504 : ffff8b14465ae680) 00000100:00100000:2.0:1787695979.384599:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465aca80 x1874531139556224/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:1.0:1787695979.384603:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ae680 x1874531139555584/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695979.384622:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695979.384624:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787695979.384625:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695979.384631:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000040:2.0:1787695979.384633:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004fad31e4] to pc [ptlrpcd_rcv+-1] req@ffff8b14465aca80 x1874531139556224/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1787695979.384637:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695979.384642:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139555584, offset 0 00000400:00000200:1.0:1787695979.384653:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695979.384654:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695979.384660:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:2.0:1787695979.384660:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695979.384668:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000040:2.0:1787695979.384668:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1787695979.384676:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:2.0:1787695979.384676:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695979.384683:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000040:2.0:1787695979.384683:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1787695979.384690:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000040:2.0:1787695979.384693:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3877) 00000400:00000200:1.0:1787695979.384695:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695979.384701:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17150:17150:256:4294967295] 192.168.201.140@tcp LPNI seq info [17149:17149:8:4294967295] 00000400:00000200:1.0:1787695979.384706:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695979.384716:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695979.384722:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695979.384732:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695979.384737:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695979.384740:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1787695979.384745:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695979.384747:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1787695979.384752:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695979.384753:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695979.384757:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695979.384760:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695979.384764:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695979.384766:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695979.384769:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695979.384773:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ae680 x1874531139555584/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.384805:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695979.384809:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ae680 x1874531139555584/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696004 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.384816:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.384817:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695979.384822:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695979.384826:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000400:00000200:1.0:1787695979.384830:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695979.384833:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298540999 00000100:00000001:1.0:1787695979.384835:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298551097 00000100:00000001:1.0:1787695979.384838:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298540999 00000100:00000001:1.0:1787695979.384840:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298551097 00000100:00000001:1.0:1787695979.384842:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695979.384844:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.384846:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695979.384850:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695979/real 1787695979] req@ffff8b14465ae680 x1874531139555584/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695979.384872:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695979.384873:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695979.384875:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695979.384878:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695979.384880:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695979.384883:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695979.384887:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ae680 x1874531139555584/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695979.384897:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ae680 x1874531139555584/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.384904:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695979.384905:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.384907:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695979.384909:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695979.384913:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ae680 x1874531139555584/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.384920:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695979.384925:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ae680 x1874531139555584/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.384932:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695979.384935:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.384937:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695979.384938:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.384940:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695979.384942:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695979.384945:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695979.384948:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695979.384951:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695979.384963:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ae680 x1874531139555584/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695979.384972:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139555584:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695979.384977:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695979.384980:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ac700 x1874531139555712/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.384987:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695979.384989:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695979.384991:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695979.384993:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695979.384995:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695979.384999:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139555712:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695979.385002:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695979.385004:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695979.385006:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695979.385008:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695979.385012:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695979.385014:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695979.385018:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695979.385021:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695979.385024:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695979.385027:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139555712, portal 10 00000100:00000001:1.0:1787695979.385029:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695979.385030:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928802048 : -128555780749568 : ffff8b14465ac700) 00000100:00000040:1.0:1787695979.385037:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ac700 x1874531139555712/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.385045:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695979.385047:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695979.385050:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695979.385052:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139555712, offset 0 00000400:00000200:1.0:1787695979.385062:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695979.385066:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695979.385070:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695979.385073:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695979.385076:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695979.385079:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695979.385081:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695979.385084:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17151:17151:256:4294967295] 192.168.201.140@tcp LPNI seq info [17150:17150:8:4294967295] 00000400:00000200:1.0:1787695979.385088:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695979.385095:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695979.385100:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695979.385104:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695979.385110:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695979.385113:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1787695979.385117:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695979.385119:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1787695979.385123:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695979.385126:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695979.385129:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695979.385133:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695979.385136:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695979.385138:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695979.385141:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695979.385145:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ac700 x1874531139555712/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.385153:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695979.385156:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ac700 x1874531139555712/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696004 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.385163:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.385165:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695979.385169:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695979.385172:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787695979.385175:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695979.385183:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298539847 00000100:00000001:1.0:1787695979.385187:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298551097 00000100:00000001:1.0:1787695979.385189:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298539847 00000100:00000001:1.0:1787695979.385191:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298551097 00000100:00000001:1.0:1787695979.385193:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695979.385195:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.385197:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695979.385202:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695979/real 1787695979] req@ffff8b14465ac700 x1874531139555712/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695979.385212:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695979.385214:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695979.385216:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695979.385218:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695979.385220:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695979.385223:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695979.385227:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ac700 x1874531139555712/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695979.385236:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ac700 x1874531139555712/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.385243:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695979.385245:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.385247:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695979.385248:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695979.385253:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ac700 x1874531139555712/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.385259:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695979.385264:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ac700 x1874531139555712/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.385270:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695979.385278:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.385280:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695979.385281:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.385283:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695979.385285:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695979.385288:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695979.385292:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695979.385295:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695979.385300:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ac700 x1874531139555712/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695979.385308:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139555712:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695979.385313:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695979.385316:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465af480 x1874531139555840/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.385324:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695979.385326:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695979.385328:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695979.385330:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695979.385332:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695979.385335:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139555840:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695979.385339:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695979.385341:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695979.385342:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695979.385344:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695979.385348:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695979.385350:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695979.385353:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695979.385356:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695979.385360:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695979.385384:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139555840, portal 4 00000100:00000001:1.0:1787695979.385392:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695979.385394:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928813696 : -128555780737920 : ffff8b14465af480) 00000100:00000040:1.0:1787695979.385401:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465af480 x1874531139555840/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.385408:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695979.385411:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695979.385414:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695979.385417:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139555840, offset 0 00000400:00000200:1.0:1787695979.385422:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695979.385427:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695979.385430:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695979.385433:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695979.385436:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695979.385439:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695979.385442:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695979.385445:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17152:17152:256:4294967295] 192.168.201.140@tcp LPNI seq info [17151:17151:8:4294967295] 00000400:00000200:1.0:1787695979.385450:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695979.385457:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695979.385462:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695979.385467:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695979.385471:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695979.385474:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1787695979.385479:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695979.385482:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1787695979.385486:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695979.385489:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695979.385492:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695979.385496:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695979.385499:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695979.385501:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695979.385504:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695979.385508:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465af480 x1874531139555840/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.385530:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695979.385533:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465af480 x1874531139555840/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696004 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.385541:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.385542:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695979.385547:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695979.385550:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695979.385554:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695979.385556:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298539878 00000100:00000001:1.0:1787695979.385559:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298551097 00000100:00000001:1.0:1787695979.385561:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298539878 00000100:00000001:1.0:1787695979.385564:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298551097 00000100:00000001:1.0:1787695979.385566:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695979.385568:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.385571:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695979.385575:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695979/real 1787695979] req@ffff8b14465af480 x1874531139555840/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695979.385585:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695979.385587:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695979.385588:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695979.385591:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695979.385592:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695979.385595:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695979.385598:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465af480 x1874531139555840/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695979.385606:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465af480 x1874531139555840/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.385613:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695979.385620:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.385622:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695979.385624:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695979.385628:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465af480 x1874531139555840/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.385636:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695979.385641:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465af480 x1874531139555840/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.385648:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695979.385651:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.385653:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695979.385654:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.385656:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695979.385658:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695979.385661:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695979.385664:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695979.385667:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695979.385672:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465af480 x1874531139555840/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695979.385681:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465af480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139555840:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695979.385686:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695979.385689:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465af800 x1874531139555968/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.385696:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695979.385698:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695979.385700:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695979.385702:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695979.385704:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695979.385715:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139555968:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695979.385718:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695979.385721:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695979.385722:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695979.385724:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695979.385728:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695979.385730:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695979.385734:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695979.385737:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695979.385741:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695979.385744:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139555968, portal 4 00000100:00000001:1.0:1787695979.385746:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695979.385748:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928814592 : -128555780737024 : ffff8b14465af800) 00000100:00000040:1.0:1787695979.385754:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465af800 x1874531139555968/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.385761:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695979.385764:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695979.385766:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695979.385769:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139555968, offset 0 00000400:00000200:1.0:1787695979.385773:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695979.385791:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695979.385796:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695979.385799:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695979.385802:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695979.385805:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695979.385807:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695979.385810:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17153:17153:256:4294967295] 192.168.201.140@tcp LPNI seq info [17152:17152:8:4294967295] 00000400:00000200:1.0:1787695979.385815:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695979.385822:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695979.385827:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695979.385831:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695979.385842:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695979.385845:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1787695979.385849:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695979.385852:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1787695979.385857:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695979.385859:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695979.385862:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695979.385866:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695979.385870:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695979.385872:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695979.385874:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695979.385878:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465af800 x1874531139555968/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.385887:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695979.385891:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465af800 x1874531139555968/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696004 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.385897:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.385899:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695979.385903:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695979.385906:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695979.385909:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695979.385912:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298539550 00000100:00000001:1.0:1787695979.385915:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298551097 00000100:00000001:1.0:1787695979.385917:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298539550 00000100:00000001:1.0:1787695979.385919:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298551097 00000100:00000001:1.0:1787695979.385921:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695979.385923:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.385925:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695979.385929:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695979/real 1787695979] req@ffff8b14465af800 x1874531139555968/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695979.385938:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695979.385945:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695979.385947:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695979.385950:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695979.385951:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695979.385954:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695979.385959:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465af800 x1874531139555968/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695979.385967:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465af800 x1874531139555968/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.385973:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695979.385975:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.385977:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695979.385978:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695979.385983:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465af800 x1874531139555968/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.385990:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695979.385994:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465af800 x1874531139555968/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.386002:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695979.386004:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.386006:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695979.386008:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.386010:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695979.386011:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695979.386014:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695979.386018:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695979.386020:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695979.386025:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465af800 x1874531139555968/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695979.386044:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139555968:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695979.386049:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695979.386053:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ad880 x1874531139556096/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.386060:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695979.386062:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695979.386064:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695979.386065:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695979.386068:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695979.386071:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139556096:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695979.386074:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695979.386076:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695979.386078:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695979.386080:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695979.386083:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695979.386085:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695979.386089:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695979.386092:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695979.386095:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695979.386098:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139556096, portal 10 00000100:00000001:1.0:1787695979.386100:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695979.386101:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928806528 : -128555780745088 : ffff8b14465ad880) 00000100:00000040:1.0:1787695979.386108:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ad880 x1874531139556096/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.386115:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695979.386118:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695979.386120:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695979.386122:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139556096, offset 0 00000400:00000200:1.0:1787695979.386126:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695979.386135:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695979.386139:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695979.386142:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695979.386144:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695979.386147:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695979.386149:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695979.386152:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17154:17154:256:4294967295] 192.168.201.140@tcp LPNI seq info [17153:17153:8:4294967295] 00000400:00000200:1.0:1787695979.386157:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695979.386163:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695979.386168:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695979.386171:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695979.386175:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695979.386178:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1787695979.386182:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695979.386190:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1787695979.386194:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695979.386196:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695979.386199:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695979.386203:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695979.386207:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695979.386209:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695979.386211:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695979.386215:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ad880 x1874531139556096/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.386223:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695979.386226:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ad880 x1874531139556096/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696004 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.386232:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.386234:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695979.386237:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695979.386240:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695979.386243:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695979.386251:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298539736 00000100:00000001:1.0:1787695979.386253:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298551097 00000100:00000001:1.0:1787695979.386255:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298539736 00000100:00000001:1.0:1787695979.386257:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298551097 00000100:00000001:1.0:1787695979.386259:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695979.386262:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.386263:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695979.386268:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695979/real 1787695979] req@ffff8b14465ad880 x1874531139556096/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695979.386277:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695979.386278:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695979.386280:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695979.386283:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695979.386284:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695979.386286:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695979.386289:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ad880 x1874531139556096/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695979.386297:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ad880 x1874531139556096/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.386303:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695979.386304:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.386306:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695979.386308:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695979.386311:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ad880 x1874531139556096/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.386317:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695979.386321:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ad880 x1874531139556096/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.386328:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695979.386335:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.386337:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695979.386338:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.386340:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695979.386342:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695979.386344:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695979.386347:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695979.386349:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695979.386353:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ad880 x1874531139556096/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695979.386360:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139556096:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695979.386385:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.386388:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695979.386392:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ae680 x1874531139555584/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.386399:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695979.386401:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695979.386403:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546000. 02000000:00000001:1.0:1787695979.386406:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695979.386408:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695979.386410:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695979.386412:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695979.386414:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661c00. 02000000:00000001:1.0:1787695979.386417:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695979.386419:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695979.386420:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ae680. 00000100:00000001:1.0:1787695979.386424:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695979.386425:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.386427:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695979.386430:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ac700 x1874531139555712/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.386448:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695979.386449:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695979.386451:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695979.386454:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695979.386456:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695979.386458:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695979.386460:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695979.386461:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445660000. 02000000:00000001:1.0:1787695979.386463:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695979.386465:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695979.386466:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ac700. 00000100:00000001:1.0:1787695979.386468:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695979.386470:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.386471:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695979.386475:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465af480 x1874531139555840/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.386482:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695979.386483:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695979.386484:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695979.386487:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695979.386488:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695979.386489:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695979.386491:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695979.386493:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445662400. 02000000:00000001:1.0:1787695979.386495:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695979.386496:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695979.386497:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465af480. 00000100:00000001:1.0:1787695979.386500:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695979.386501:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.386503:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695979.386507:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465af800 x1874531139555968/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.386514:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695979.386516:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695979.386522:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695979.386526:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695979.386528:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695979.386529:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695979.386531:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695979.386533:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661800. 02000000:00000001:1.0:1787695979.386535:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695979.386536:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695979.386538:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465af800. 00000100:00000001:1.0:1787695979.386540:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695979.386541:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.386542:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695979.386545:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ad880 x1874531139556096/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.386549:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695979.386550:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695979.386551:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695979.386553:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695979.386553:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695979.386554:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695979.386555:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695979.386556:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663c00. 02000000:00000001:1.0:1787695979.386557:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695979.386558:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695979.386559:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ad880. 00000100:00000001:1.0:1787695979.386560:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695979.386561:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.386562:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.386565:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695979.386566:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.386568:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695979.386569:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695979.386570:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695979.386572:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465aca80 x1874531139556224/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.386576:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695979.386580:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695979.386582:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695979.386583:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695979.386584:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695979.386587:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139556224:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695979.386589:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695979.386590:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695979.386592:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695979.386593:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695979.386595:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695979.386596:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695979.386598:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695979.386600:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695979.386603:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695979.386605:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139556224, portal 10 00000100:00000001:1.0:1787695979.386606:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695979.386607:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928802944 : -128555780748672 : ffff8b14465aca80) 00000100:00000040:1.0:1787695979.386611:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465aca80 x1874531139556224/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.386616:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695979.386617:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695979.386619:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695979.386621:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139556224, offset 0 00000400:00000200:1.0:1787695979.386623:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695979.386626:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695979.386628:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695979.386630:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695979.386632:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695979.386634:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695979.386635:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695979.386638:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17155:17155:256:4294967295] 192.168.201.140@tcp LPNI seq info [17154:17154:8:4294967295] 00000400:00000200:1.0:1787695979.386641:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695979.386645:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695979.386652:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695979.386654:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695979.386657:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695979.386659:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000200:1.1:1787695979.386661:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695979.386663:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 22 secs later) 00000800:00000100:1.0:1787695979.386666:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695979.386667:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695979.386669:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695979.386672:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695979.386674:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695979.386675:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695979.386677:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695979.386679:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465aca80 x1874531139556224/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696004 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.386684:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695979.386686:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465aca80 x1874531139556224/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696004 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.386690:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.386692:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695979.386694:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695979.386696:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695979.386698:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695979.386699:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298539715 00000100:00000001:1.0:1787695979.386701:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298551097 00000100:00000001:1.0:1787695979.386702:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298539715 00000100:00000001:1.0:1787695979.386704:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298551097 00000100:00000001:1.0:1787695979.386706:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695979.386707:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.386708:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695979.386711:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695979/real 1787695979] req@ffff8b14465aca80 x1874531139556224/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695979.386720:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695979.386721:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695979.386722:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695979.386724:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695979.386725:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695979.386726:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695979.386728:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465aca80 x1874531139556224/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695979.386733:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465aca80 x1874531139556224/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.386737:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695979.386738:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.386739:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695979.386740:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695979.386742:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465aca80 x1874531139556224/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.386746:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695979.386748:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465aca80 x1874531139556224/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.386752:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695979.386754:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.386755:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695979.386756:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.386757:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695979.386758:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695979.386760:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695979.386762:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695979.386763:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695979.386771:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465aca80 x1874531139556224/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695979.386793:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139556224:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695979.386798:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.386800:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695979.386803:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465aca80 x1874531139556224/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696004 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695979.386810:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695979.386812:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695979.386814:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695979.386817:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695979.386818:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695979.386820:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695979.386822:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695979.386823:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445662c00. 02000000:00000001:1.0:1787695979.386826:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695979.386827:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695979.386829:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465aca80. 00000100:00000001:1.0:1787695979.386831:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695979.386833:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.386834:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695979.386838:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695979.386840:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695979.386842:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695979.386844:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695979.446477:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695979.446484:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695979.446485:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695979.446490:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695979.446492:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695979.446496:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695979.446497:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695979.446499:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695979.446501:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695980.086471:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695980.086477:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695980.086482:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695980.086483:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695980.086486:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695980.086487:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695980.086489:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695980.086490:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695980.086492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695980.086511:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695980.086514:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695980.086514:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695980.086589:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695980.086594:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695980.086598:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695980.470443:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695980.470449:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695980.470451:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695980.470456:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695980.470458:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695980.470461:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695980.470463:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695980.470465:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695980.470467:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695981.110485:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695981.110492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695981.110498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695981.110500:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695981.110504:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695981.110505:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695981.110509:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695981.110511:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695981.110515:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695981.110516:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695981.110520:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695981.110521:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695981.110580:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695981.110583:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695981.110586:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695981.494464:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695981.494470:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695981.494472:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695981.494476:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695981.494479:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695981.494482:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695981.494484:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695981.494486:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695981.494488:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695982.134467:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695982.134473:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695982.134477:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695982.134478:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695982.134480:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695982.134481:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695982.134484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695982.134485:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695982.134487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695982.134490:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695982.134492:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695982.134496:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695982.134498:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695982.134499:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695982.134502:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695982.134503:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695982.134505:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695982.134507:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695982.134509:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695982.134510:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695982.134556:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695982.134560:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695982.134563:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695982.518426:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695982.518433:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695982.518435:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695982.518439:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695982.518442:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695982.518445:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695982.518455:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695982.518457:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695982.518476:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695983.158472:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695983.158478:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695983.158482:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695983.158500:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695983.158503:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695983.158504:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695983.158506:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695983.158507:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695983.158510:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695983.158511:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695983.158513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695983.158514:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695983.158599:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695983.158603:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695983.158606:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695983.542458:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695983.542465:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695983.542468:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695983.542474:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695983.542478:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695983.542482:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695983.542485:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695983.542487:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695983.542489:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695984.182458:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695984.182464:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695984.182466:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695984.182468:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695984.182473:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695984.182475:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695984.182476:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695984.182478:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695984.182481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695984.182482:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695984.182483:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695984.182484:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695984.182486:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695984.182487:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695984.182488:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695984.182489:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695984.182492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695984.182493:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695984.182495:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695984.182496:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695984.182498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695984.182499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695984.182501:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695984.182502:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695984.182592:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695984.182599:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695984.182604:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787695984.502754:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695984.502767:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787695984.502770:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695984.502774:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695984.502779:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695984.502810:0:47074:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3872 00000400:00000200:2.0:1787695984.502820:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695984.502829:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695984.502834:0:47074:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695984.502838:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.502843:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695984.502846:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.502849:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695984.502854:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695984.502857:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:2.0:1787695984.502861:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695984.502865:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:2.0:1787695984.502868:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:2.0:1787695984.502871:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:2.0:1787695984.502875:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.502877:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695984.502879:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.502881:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695984.502884:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:2.0:1787695984.502887:0:47074:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695984.502890:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695984.502894:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695984.502900:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695984.502903:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695984.502906:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787695984.502909:0:47074:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695984.502918:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ad500. 00000020:00000040:2.0:1787695984.502923:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:2.0:1787695984.502928:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695984.502930:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695984.502934:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695984.502937:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695984.502948:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695984.502957:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663800. 00000100:00000001:2.0:1787695984.502965:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695984.502976:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ad500 x1874531139556352/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695984.502986:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695984.502990:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695984.502995:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004ab396cc] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ad500 x1874531139556352/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695984.503015:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695984.503018:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695984.503020:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787695984.503021:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695984.503023:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695984.503025:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695984.503027:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3872 00000400:00000200:2.0:1787695984.503030:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695984.503033:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695984.503035:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695984.503039:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.503042:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695984.503044:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.503047:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695984.503049:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695984.503050:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:2.0:1787695984.503051:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695984.503053:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000100:00000001:1.0:1787695984.503055:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787695984.503056:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:2.0:1787695984.503058:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000001:1.0:1787695984.503064:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1787695984.503066:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695984.503070:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787695984.503077:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695984.503088:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695984.503089:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ad500 x1874531139556352/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695984.503097:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695984.503108:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:2.0:1787695984.503121:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695984.503124:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695984.503125:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695984.503131:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695984.503133:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695984.503142:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695984.503148:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787695984.503153:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695984.503162:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695984.503168:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000010:2.0:1787695984.503170:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465afb80. 00000100:00000001:1.0:1787695984.503171:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1787695984.503176:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:2.0:1787695984.503180:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695984.503181:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695984.503182:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695984.503187:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695984.503188:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695984.503190:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695984.503191:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695984.503198:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445662c00. 02000000:00000001:1.0:1787695984.503199:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695984.503204:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695984.503216:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465afb80 x1874531139556480/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787695984.503218:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139556352:192.168.201.140@tcp:250:kworker.0 00000100:00000001:1.0:1787695984.503241:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787695984.503243:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695984.503247:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695984.503250:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:2.0:1787695984.503254:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b9acb62b] to pc [ptlrpcd_rcv+-1] req@ffff8b14465afb80 x1874531139556480/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695984.503255:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695984.503262:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695984.503272:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 00000100:00000001:2.0:1787695984.503274:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695984.503280:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695984.503282:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695984.503288:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:1.0:1787695984.503293:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695984.503298:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695984.503301:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:1.0:1787695984.503302:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000100:00000001:2.0:1787695984.503311:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:1.0:1787695984.503315:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00080000:2.0:1787695984.503317:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3872 00000100:00000200:1.0:1787695984.503325:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139556352, portal 25 00000400:00000200:2.0:1787695984.503331:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695984.503332:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695984.503336:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928805632 : -128555780745984 : ffff8b14465ad500) 00000400:00000200:2.0:1787695984.503342:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000040:1.0:1787695984.503344:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ad500 x1874531139556352/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696009 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695984.503349:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695984.503357:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.503402:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695984.503411:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.503422:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695984.503425:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1787695984.503427:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695984.503429:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000100:00000040:1.0:1787695984.503433:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000020:00000001:2.0:1787695984.503437:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000010:1.0:1787695984.503441:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000020:00000040:2.0:1787695984.503441:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:2.0:1787695984.503450:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000200:1.0:1787695984.503451:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139556352, offset 0 00000020:00000001:2.0:1787695984.503453:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000400:00000200:1.0:1787695984.503460:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:2.0:1787695984.503465:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.503471:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695984.503475:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:2.0:1787695984.503480:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.503487:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695984.503489:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000020:00000040:2.0:1787695984.503498:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000400:00000200:1.0:1787695984.503499:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695984.503506:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00080000:2.0:1787695984.503512:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:1.0:1787695984.503516:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695984.503523:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.503529:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695984.503535:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1787695984.503536:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17156:17156:256:4294967295] 192.168.201.140@tcp LPNI seq info [17155:17155:8:4294967295] 00000400:00000200:1.0:1787695984.503544:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:2.0:1787695984.503546:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695984.503552:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695984.503561:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695984.503564:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1787695984.503566:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000008:00000001:2.0:1787695984.503568:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695984.503577:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695984.503581:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ad180. 00000800:00000200:1.0:1787695984.503582:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000020:00000040:2.0:1787695984.503585:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:2.0:1787695984.503590:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695984.503591:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1787695984.503592:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000100:00000001:2.0:1787695984.503593:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695984.503596:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695984.503599:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695984.503605:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663c00. 00000100:00000001:2.0:1787695984.503611:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695984.503613:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00100000:2.0:1787695984.503619:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ad180 x1874531139556608/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1787695984.503622:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1787695984.503629:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695984.503632:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1787695984.503639:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:2.0:1787695984.503643:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006ad773a4] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ad180 x1874531139556608/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1787695984.503644:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1787695984.503659:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695984.503664:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695984.503675:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1787695984.503679:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695984.503687:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00080000:2.0:1787695984.503688:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695984.503705:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787695984.503709:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695984.503710:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1787695984.503712:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000100:00000001:2.0:1787695984.503723:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:1.0:1787695984.503724:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00080000:2.0:1787695984.503731:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3872 00000100:00000001:1.0:1787695984.503732:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000400:00000200:2.0:1787695984.503740:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000200:1.0:1787695984.503742:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ad500 x1874531139556352/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696009 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787695984.503747:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695984.503755:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695984.503766:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.503769:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695984.503771:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.503773:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695984.503776:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695984.503778:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695984.503784:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ad500 x1874531139556352/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696009 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1787695984.503785:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:2.0:1787695984.503791:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695984.503799:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:2.0:1787695984.503810:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:2.0:1787695984.503820:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:1.0:1787695984.503821:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695984.503830:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:2.0:1787695984.503833:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.503836:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695984.503842:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695984.503843:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695984.503853:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787695984.503860:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000040:2.0:1787695984.503860:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00000001:1.0:1787695984.503870:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298556217 00000100:00080000:2.0:1787695984.503873:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695984.503880:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695984.503883:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695984.503886:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298551097 02000000:00000001:2.0:1787695984.503887:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695984.503891:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695984.503892:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298556217 00000100:00000001:2.0:1787695984.503894:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695984.503900:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787695984.503901:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298551097 00000008:00000001:2.0:1787695984.503915:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695984.503917:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695984.503922:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695984.503923:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695984.503928:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465adc00. 00000020:00000040:2.0:1787695984.503932:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000100:00000001:1.0:1787695984.503934:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1787695984.503935:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695984.503938:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695984.503944:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:1.0:1787695984.503945:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695984/real 1787695984] req@ffff8b14465ad500 x1874531139556352/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695984.503947:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695984.503951:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695984.503963:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661800. 00000400:00000001:1.0:1787695984.503966:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1787695984.503968:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787695984.503970:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695984.503973:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000100:00100000:2.0:1787695984.503977:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465adc00 x1874531139556736/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695984.503978:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695984.503982:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695984.503990:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787695984.503995:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000200:1.0:1787695984.503997:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ad500 x1874531139556352/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695984.503998:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695984.504006:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ad500 x1874531139556352/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695984.504020:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000114811d4] to pc [ptlrpcd_rcv+-1] req@ffff8b14465adc00 x1874531139556736/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.504021:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695984.504025:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.504033:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695984.504037:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695984.504049:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695984.504053:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ad500 x1874531139556352/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695984.504055:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695984.504070:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1787695984.504081:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787695984.504082:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787695984.504089:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1787695984.504096:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ad500 x1874531139556352/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695984.504097:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695984.504106:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3872 00000400:00000200:2.0:1787695984.504125:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695984.504131:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:2.0:1787695984.504135:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695984.504139:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787695984.504144:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695984.504145:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695984.504149:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695984.504152:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.504157:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695984.504158:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000040:2.0:1787695984.504161:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.504164:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695984.504165:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1787695984.504166:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695984.504167:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:2.0:1787695984.504169:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695984.504173:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000100:00080000:1.0:1787695984.504177:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:2.0:1787695984.504177:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:2.0:1787695984.504181:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:2.0:1787695984.504189:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.504191:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:1.0:1787695984.504196:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000040:2.0:1787695984.504196:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.504200:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695984.504201:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:2.0:1787695984.504203:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695984.504208:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695984.504213:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1787695984.504217:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695984.504221:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695984.504225:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695984.504227:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ad500 x1874531139556352/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695984.504228:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695984.504232:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787695984.504240:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695984.504247:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695984.504258:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ae300. 00000100:00100000:1.0:1787695984.504268:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139556352:192.168.201.140@tcp:250:kworker.0 00000020:00000040:2.0:1787695984.504268:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:2.0:1787695984.504279:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787695984.504283:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787695984.504283:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695984.504292:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787695984.504294:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1787695984.504296:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695984.504301:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695984.504302:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ad500 x1874531139556352/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1787695984.504311:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445662400. 00000100:00000001:1.0:1787695984.504321:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1787695984.504322:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695984.504327:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695984.504330:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 00000100:00100000:2.0:1787695984.504335:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ae300 x1874531139556864/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787695984.504337:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695984.504340:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695984.504343:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:1.0:1787695984.504354:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695984.504358:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663800. 02000000:00000001:1.0:1787695984.504386:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1787695984.504392:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787695984.504393:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695984.504398:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ad500. 00000100:00000001:2.0:1787695984.504400:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695984.504403:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695984.504406:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.504410:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695984.504412:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004515329a] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ae300 x1874531139556864/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.504418:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695984.504420:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695984.504426:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695984.504430:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695984.504438:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787695984.504442:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695984.504445:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465afb80 x1874531139556480/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695984.504449:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695984.504456:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695984.504461:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695984.504461:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 02000000:00000001:1.0:1787695984.504465:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787695984.504467:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1787695984.504468:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787695984.504472:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1787695984.504482:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695984.504484:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695984.504490:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3872 00000100:00100000:1.0:1787695984.504491:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139556480:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695984.504504:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000400:00000200:2.0:1787695984.504504:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:1.0:1787695984.504509:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000400:00000200:2.0:1787695984.504513:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 02000000:00000001:1.0:1787695984.504514:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695984.504521:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1787695984.504524:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 02000000:00000010:1.0:1787695984.504526:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695984.504527:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695984.504531:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695984.504533:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695984.504541:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695984.504545:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139556480, portal 10 00000100:00000040:2.0:1787695984.504546:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695984.504548:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695984.504552:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928815488 : -128555780736128 : ffff8b14465afb80) 00000100:00000001:2.0:1787695984.504554:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695984.504560:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.504564:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000040:1.0:1787695984.504565:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465afb80 x1874531139556480/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696009 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787695984.504566:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695984.504569:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:2.0:1787695984.504572:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695984.504577:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000020:00000001:2.0:1787695984.504584:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000020:00000001:2.0:1787695984.504588:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000040:2.0:1787695984.504597:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.504603:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695984.504605:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695984.504606:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.504612:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695984.504614:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00000040:1.0:1787695984.504620:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00080000:2.0:1787695984.504622:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000010:1.0:1787695984.504629:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000001:2.0:1787695984.504629:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695984.504641:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:1.0:1787695984.504642:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139556480, offset 0 00000400:00000200:1.0:1787695984.504656:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 02000000:00000001:2.0:1787695984.504656:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695984.504667:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695984.504669:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695984.504679:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:1.0:1787695984.504680:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000008:00100000:2.0:1787695984.504682:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1787695984.504686:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000008:00000001:2.0:1787695984.504687:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695984.504704:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695984.504707:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000010:2.0:1787695984.504711:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465aed80. 00000400:00000200:1.0:1787695984.504712:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000020:00000040:2.0:1787695984.504718:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000400:00000200:1.0:1787695984.504721:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 02000000:00000001:2.0:1787695984.504728:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695984.504732:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695984.504736:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17157:17157:256:4294967295] 192.168.201.140@tcp LPNI seq info [17156:17156:8:4294967295] 00000100:00000001:2.0:1787695984.504738:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695984.504747:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1787695984.504753:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:2.0:1787695984.504754:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695984.504764:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445660000. 00000400:00000200:1.0:1787695984.504767:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695984.504776:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787695984.504779:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1787695984.504791:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000100:00100000:2.0:1787695984.504793:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465aed80 x1874531139556992/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1787695984.504803:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695984.504811:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000100:00000001:2.0:1787695984.504818:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695984.504821:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1787695984.504822:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:2.0:1787695984.504831:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bed91c58] to pc [ptlrpcd_rcv+-1] req@ffff8b14465aed80 x1874531139556992/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1787695984.504832:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1787695984.504852:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695984.504862:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695984.504875:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1787695984.504875:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695984.504889:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695984.504894:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00080000:2.0:1787695984.504899:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000200:1.0:1787695984.504902:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000100:00000001:2.0:1787695984.504904:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000010:1.0:1787695984.504906:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00080000:2.0:1787695984.504908:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695984.504915:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787695984.504916:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695984.504921:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3872 00000100:00000200:1.0:1787695984.504923:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465afb80 x1874531139556480/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696009 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787695984.504937:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695984.504950:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695984.504958:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695984.504959:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695984.504972:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695984.504973:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465afb80 x1874531139556480/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696009 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695984.504982:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695984.504991:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695984.505000:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695984.505008:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787695984.505008:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695984.505014:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000100:00000001:1.0:1787695984.505016:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1787695984.505022:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695984.505025:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1787695984.505026:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:2.0:1787695984.505031:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:1.0:1787695984.505033:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000020:00000001:2.0:1787695984.505036:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000400:00000200:1.0:1787695984.505047:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:2.0:1787695984.505049:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695984.505051:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298546119 00000100:00000001:2.0:1787695984.505057:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695984.505058:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298556217 00000100:00000001:1.0:1787695984.505061:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298546119 00000100:00000001:1.0:1787695984.505062:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298556217 00000100:00000001:1.0:1787695984.505072:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1787695984.505073:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695984.505087:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695984.505087:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695984.505097:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000040:2.0:1787695984.505098:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00080000:2.0:1787695984.505109:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00100000:1.0:1787695984.505112:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695984/real 1787695984] req@ffff8b14465afb80 x1874531139556480/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695984.505113:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695984.505121:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695984.505126:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695984.505128:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695984.505130:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695984.505134:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787695984.505146:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695984.505154:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695984.505164:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465aea00. 00000020:00000040:2.0:1787695984.505170:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:2.0:1787695984.505173:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000001:1.0:1787695984.505175:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1787695984.505177:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787695984.505181:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787695984.505181:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695984.505187:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695984.505189:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695984.505190:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 02000000:00000010:2.0:1787695984.505194:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661c00. 00000400:00000200:1.0:1787695984.505197:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000100:00000001:2.0:1787695984.505198:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695984.505206:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00100000:2.0:1787695984.505208:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465aea00 x1874531139557120/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695984.505215:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787695984.505220:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695984.505223:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695984.505234:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a41ef0cb] to pc [ptlrpcd_rcv+-1] req@ffff8b14465aea00 x1874531139557120/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695984.505235:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465afb80 x1874531139556480/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695984.505274:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695984.505280:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1787695984.505287:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465afb80 x1874531139556480/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695984.505288:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1787695984.505295:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695984.505300:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695984.505305:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.505308:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:2.0:1787695984.505308:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1787695984.505316:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695984.505320:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3882) 00000100:00100000:1.0:1787695984.505327:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465afb80 x1874531139556480/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.505334:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695984.505337:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465afb80 x1874531139556480/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.505342:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695984.505346:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.505348:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695984.505349:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.505350:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695984.505352:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695984.505354:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695984.505356:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695984.505358:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695984.505388:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465afb80 x1874531139556480/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695984.505395:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139556480:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695984.505398:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695984.505401:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ad180 x1874531139556608/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.505407:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695984.505408:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695984.505410:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695984.505412:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695984.505413:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695984.505415:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139556608:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695984.505417:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695984.505420:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695984.505421:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695984.505422:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695984.505425:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695984.505427:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695984.505430:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695984.505432:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695984.505434:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695984.505436:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139556608, portal 10 00000100:00000001:1.0:1787695984.505438:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695984.505439:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928804736 : -128555780746880 : ffff8b14465ad180) 00000100:00000040:1.0:1787695984.505443:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ad180 x1874531139556608/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696009 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.505447:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695984.505449:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695984.505451:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695984.505452:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139556608, offset 0 00000400:00000200:1.0:1787695984.505459:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695984.505462:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695984.505465:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695984.505467:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695984.505469:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695984.505471:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695984.505473:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695984.505475:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17158:17158:256:4294967295] 192.168.201.140@tcp LPNI seq info [17157:17157:8:4294967295] 00000400:00000200:1.0:1787695984.505478:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695984.505483:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695984.505486:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695984.505488:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695984.505491:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695984.505494:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1787695984.505497:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695984.505499:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1787695984.505501:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695984.505502:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695984.505505:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695984.505507:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695984.505509:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695984.505510:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695984.505512:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695984.505515:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ad180 x1874531139556608/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696009 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.505520:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695984.505522:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ad180 x1874531139556608/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696009 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.505527:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695984.505528:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695984.505530:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695984.505532:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787695984.505534:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695984.505540:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298544968 00000100:00000001:1.0:1787695984.505542:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298556217 00000100:00000001:1.0:1787695984.505544:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298544968 00000100:00000001:1.0:1787695984.505545:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298556217 00000100:00000001:1.0:1787695984.505547:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695984.505549:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695984.505550:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695984.505553:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695984/real 1787695984] req@ffff8b14465ad180 x1874531139556608/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695984.505558:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695984.505559:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695984.505561:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695984.505563:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695984.505565:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695984.505567:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695984.505570:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ad180 x1874531139556608/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695984.505579:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ad180 x1874531139556608/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.505586:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695984.505588:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.505590:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695984.505592:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695984.505595:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ad180 x1874531139556608/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.505603:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695984.505607:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ad180 x1874531139556608/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.505613:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695984.505620:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.505621:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695984.505622:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.505623:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695984.505624:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695984.505626:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695984.505629:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695984.505630:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695984.505633:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ad180 x1874531139556608/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695984.505639:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139556608:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695984.505641:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695984.505644:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465adc00 x1874531139556736/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.505648:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695984.505649:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695984.505650:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695984.505651:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695984.505653:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695984.505655:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139556736:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695984.505657:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695984.505659:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695984.505660:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695984.505661:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695984.505663:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695984.505665:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695984.505668:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695984.505669:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695984.505672:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695984.505674:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139556736, portal 4 00000100:00000001:1.0:1787695984.505678:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695984.505679:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928807424 : -128555780744192 : ffff8b14465adc00) 00000100:00000040:1.0:1787695984.505684:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465adc00 x1874531139556736/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696009 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.505701:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695984.505704:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695984.505706:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695984.505708:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139556736, offset 0 00000400:00000200:1.0:1787695984.505711:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695984.505714:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695984.505717:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695984.505719:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695984.505721:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695984.505723:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695984.505724:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695984.505726:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17159:17159:256:4294967295] 192.168.201.140@tcp LPNI seq info [17158:17158:8:4294967295] 00000400:00000200:1.0:1787695984.505729:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695984.505733:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695984.505736:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695984.505738:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695984.505741:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695984.505743:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1787695984.505746:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695984.505747:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1787695984.505750:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695984.505751:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695984.505753:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695984.505756:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695984.505758:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695984.505759:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695984.505761:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695984.505763:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465adc00 x1874531139556736/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696009 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.505777:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695984.505779:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465adc00 x1874531139556736/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696009 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.505784:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695984.505785:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695984.505788:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695984.505790:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695984.505793:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695984.505794:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298544998 00000100:00000001:1.0:1787695984.505796:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298556217 00000100:00000001:1.0:1787695984.505797:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298544998 00000100:00000001:1.0:1787695984.505799:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298556217 00000100:00000001:1.0:1787695984.505800:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695984.505801:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695984.505803:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695984.505805:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695984/real 1787695984] req@ffff8b14465adc00 x1874531139556736/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695984.505810:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695984.505810:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695984.505812:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695984.505813:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695984.505814:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695984.505816:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695984.505818:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465adc00 x1874531139556736/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695984.505823:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465adc00 x1874531139556736/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.505827:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695984.505830:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.505832:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695984.505833:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695984.505835:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465adc00 x1874531139556736/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.505840:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695984.505842:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465adc00 x1874531139556736/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.505847:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695984.505849:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.505849:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695984.505850:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.505851:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695984.505852:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695984.505854:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695984.505856:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695984.505857:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695984.505859:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465adc00 x1874531139556736/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695984.505864:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139556736:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695984.505866:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.505868:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695984.505870:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465afb80 x1874531139556480/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.505874:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695984.505876:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695984.505877:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695984.505879:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695984.505880:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695984.505884:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695984.505885:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695984.505887:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445662c00. 02000000:00000001:1.0:1787695984.505889:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695984.505890:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695984.505891:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465afb80. 00000100:00000001:1.0:1787695984.505893:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695984.505895:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.505896:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695984.505898:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ad180 x1874531139556608/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.505902:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695984.505903:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695984.505904:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695984.505905:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695984.505906:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695984.505907:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695984.505908:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695984.505908:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663c00. 02000000:00000001:1.0:1787695984.505910:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695984.505910:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695984.505911:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ad180. 00000100:00000001:1.0:1787695984.505912:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695984.505913:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.505914:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695984.505916:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465adc00 x1874531139556736/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.505919:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695984.505920:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695984.505921:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695984.505923:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695984.505923:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695984.505924:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695984.505925:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695984.505926:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661800. 02000000:00000001:1.0:1787695984.505927:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695984.505931:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695984.505932:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465adc00. 00000100:00000001:1.0:1787695984.505933:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695984.505934:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.505935:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.505937:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695984.505938:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695984.505940:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695984.505941:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695984.505942:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695984.505945:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ae300 x1874531139556864/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.505949:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695984.505950:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695984.505952:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695984.505953:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695984.505955:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695984.505956:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139556864:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695984.505958:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695984.505960:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695984.505961:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695984.505962:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695984.505964:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695984.505965:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695984.505967:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695984.505968:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695984.505971:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695984.505973:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139556864, portal 4 00000100:00000001:1.0:1787695984.505974:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695984.505975:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928809216 : -128555780742400 : ffff8b14465ae300) 00000100:00000040:1.0:1787695984.505979:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ae300 x1874531139556864/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696009 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.505984:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695984.505986:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695984.505991:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695984.505993:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139556864, offset 0 00000400:00000200:1.0:1787695984.505995:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695984.505998:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695984.506000:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695984.506003:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695984.506005:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695984.506006:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695984.506008:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695984.506010:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17160:17160:256:4294967295] 192.168.201.140@tcp LPNI seq info [17159:17159:8:4294967295] 00000400:00000200:1.0:1787695984.506012:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695984.506017:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695984.506020:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695984.506022:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695984.506024:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695984.506026:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1787695984.506029:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695984.506031:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1787695984.506034:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695984.506035:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695984.506037:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695984.506039:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695984.506041:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695984.506043:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695984.506044:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695984.506047:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ae300 x1874531139556864/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696009 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506052:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695984.506054:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ae300 x1874531139556864/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696009 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506058:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695984.506059:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695984.506064:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695984.506066:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695984.506069:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695984.506070:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298544670 00000100:00000001:1.0:1787695984.506072:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298556217 00000100:00000001:1.0:1787695984.506074:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298544670 00000100:00000001:1.0:1787695984.506084:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298556217 00000100:00000001:1.0:1787695984.506085:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695984.506090:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695984.506091:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695984.506094:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695984/real 1787695984] req@ffff8b14465ae300 x1874531139556864/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695984.506099:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695984.506100:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695984.506102:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695984.506103:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695984.506104:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695984.506106:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695984.506108:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ae300 x1874531139556864/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695984.506113:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ae300 x1874531139556864/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506117:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695984.506118:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.506119:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695984.506120:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695984.506122:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ae300 x1874531139556864/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506126:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695984.506128:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ae300 x1874531139556864/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506137:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695984.506138:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.506140:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695984.506141:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.506142:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695984.506143:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695984.506144:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695984.506147:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695984.506148:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695984.506150:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ae300 x1874531139556864/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695984.506156:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139556864:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695984.506158:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695984.506161:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465aed80 x1874531139556992/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506165:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695984.506166:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695984.506167:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695984.506168:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695984.506169:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695984.506171:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139556992:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695984.506173:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695984.506174:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695984.506175:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695984.506177:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695984.506178:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695984.506179:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695984.506182:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695984.506186:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695984.506188:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695984.506190:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139556992, portal 10 00000100:00000001:1.0:1787695984.506192:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695984.506193:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928811904 : -128555780739712 : ffff8b14465aed80) 00000100:00000040:1.0:1787695984.506197:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465aed80 x1874531139556992/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696009 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506202:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695984.506203:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695984.506204:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695984.506206:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139556992, offset 0 00000400:00000200:1.0:1787695984.506208:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695984.506211:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695984.506213:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695984.506215:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695984.506217:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695984.506218:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695984.506220:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695984.506222:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17161:17161:256:4294967295] 192.168.201.140@tcp LPNI seq info [17160:17160:8:4294967295] 00000400:00000200:1.0:1787695984.506225:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695984.506229:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695984.506231:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695984.506233:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695984.506236:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695984.506237:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1787695984.506240:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695984.506242:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1787695984.506245:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695984.506247:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695984.506250:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695984.506253:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695984.506256:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695984.506258:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695984.506265:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695984.506270:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465aed80 x1874531139556992/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696009 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506278:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695984.506281:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465aed80 x1874531139556992/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696009 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506288:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695984.506290:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695984.506294:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695984.506297:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695984.506300:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695984.506303:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298544856 00000100:00000001:1.0:1787695984.506305:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298556217 00000100:00000001:1.0:1787695984.506307:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298544856 00000100:00000001:1.0:1787695984.506309:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298556217 00000100:00000001:1.0:1787695984.506311:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695984.506313:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695984.506316:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695984.506319:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695984/real 1787695984] req@ffff8b14465aed80 x1874531139556992/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695984.506326:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695984.506328:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695984.506330:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695984.506332:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695984.506333:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695984.506335:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695984.506338:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465aed80 x1874531139556992/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695984.506345:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465aed80 x1874531139556992/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506361:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695984.506396:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.506399:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695984.506401:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695984.506407:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465aed80 x1874531139556992/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506415:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695984.506420:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465aed80 x1874531139556992/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506428:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695984.506431:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.506433:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695984.506434:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.506436:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695984.506438:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695984.506440:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695984.506443:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695984.506445:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695984.506450:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465aed80 x1874531139556992/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695984.506455:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139556992:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695984.506459:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695984.506461:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465aea00 x1874531139557120/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506465:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695984.506466:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695984.506472:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695984.506473:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695984.506475:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695984.506477:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139557120:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695984.506480:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695984.506481:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695984.506482:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695984.506484:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695984.506486:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695984.506487:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695984.506490:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695984.506491:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695984.506494:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695984.506496:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139557120, portal 10 00000100:00000001:1.0:1787695984.506497:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695984.506498:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928811008 : -128555780740608 : ffff8b14465aea00) 00000100:00000040:1.0:1787695984.506502:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465aea00 x1874531139557120/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696009 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506506:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695984.506508:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695984.506509:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695984.506511:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139557120, offset 0 00000400:00000200:1.0:1787695984.506513:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695984.506516:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695984.506518:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695984.506520:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695984.506522:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695984.506524:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695984.506526:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695984.506528:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17162:17162:256:4294967295] 192.168.201.140@tcp LPNI seq info [17161:17161:8:4294967295] 00000400:00000200:1.0:1787695984.506531:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695984.506535:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695984.506543:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695984.506545:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695984.506548:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695984.506550:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000800:00000200:1.1:1787695984.506552:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695984.506554:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 17 secs later) 00000800:00000100:1.0:1787695984.506557:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695984.506558:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695984.506575:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695984.506579:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695984.506583:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695984.506585:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695984.506588:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695984.506592:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465aea00 x1874531139557120/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696009 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506600:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695984.506604:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465aea00 x1874531139557120/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696009 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506611:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695984.506612:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695984.506615:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695984.506617:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695984.506619:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695984.506621:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298544835 00000100:00000001:1.0:1787695984.506623:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298556217 00000100:00000001:1.0:1787695984.506624:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298544835 00000100:00000001:1.0:1787695984.506626:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298556217 00000100:00000001:1.0:1787695984.506627:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695984.506629:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695984.506630:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695984.506632:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695984/real 1787695984] req@ffff8b14465aea00 x1874531139557120/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695984.506643:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695984.506644:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695984.506645:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695984.506647:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695984.506648:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695984.506649:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695984.506651:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465aea00 x1874531139557120/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695984.506657:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465aea00 x1874531139557120/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506661:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695984.506662:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.506663:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695984.506664:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695984.506666:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465aea00 x1874531139557120/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506670:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695984.506672:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465aea00 x1874531139557120/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506677:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695984.506679:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.506680:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695984.506680:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.506682:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695984.506683:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695984.506684:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695984.506686:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695984.506687:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695984.506709:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465aea00 x1874531139557120/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695984.506714:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139557120:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695984.506717:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.506719:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695984.506720:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ae300 x1874531139556864/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506724:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695984.506726:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695984.506727:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695984.506729:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695984.506730:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695984.506731:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695984.506732:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695984.506734:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445662400. 02000000:00000001:1.0:1787695984.506736:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695984.506737:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695984.506739:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ae300. 00000100:00000001:1.0:1787695984.506741:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695984.506743:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.506744:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695984.506747:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465aed80 x1874531139556992/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506754:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695984.506755:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695984.506757:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695984.506759:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695984.506761:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695984.506762:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695984.506764:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695984.506765:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445660000. 02000000:00000001:1.0:1787695984.506768:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695984.506774:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695984.506776:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465aed80. 00000100:00000001:1.0:1787695984.506778:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695984.506780:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.506782:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695984.506786:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465aea00 x1874531139557120/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696009 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695984.506792:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695984.506794:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695984.506795:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695984.506798:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695984.506799:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695984.506801:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695984.506802:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695984.506804:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661c00. 02000000:00000001:1.0:1787695984.506806:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695984.506807:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695984.506809:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465aea00. 00000100:00000001:1.0:1787695984.506811:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695984.506812:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.506814:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695984.506818:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695984.506820:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695984.506821:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695984.506823:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695984.566469:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695984.566475:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695984.566477:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695984.566481:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695984.566484:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695984.566487:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695984.566489:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695984.566491:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695984.566493:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695985.206504:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695985.206511:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695985.206517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695985.206519:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695985.206523:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695985.206524:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695985.206528:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695985.206529:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695985.206533:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695985.206534:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695985.206538:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695985.206539:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695985.206611:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695985.206615:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695985.206618:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695985.590466:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695985.590475:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695985.590478:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695985.590486:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695985.590490:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695985.590494:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695985.590498:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695985.590501:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695985.590504:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695986.230447:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695986.230455:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695986.230461:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695986.230461:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695986.230465:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695986.230468:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695986.230471:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695986.230479:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695986.230483:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695986.230485:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695986.230488:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695986.230491:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695986.230494:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695986.230497:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695986.230502:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695986.230504:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695986.230505:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695986.230507:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695986.230509:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695986.230510:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695986.230512:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695986.230514:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695986.230516:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695986.230518:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695986.230523:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695986.230524:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695986.230529:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695986.230531:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695986.230532:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695986.230534:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695986.230535:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695986.230536:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695986.230538:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695986.230539:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695986.230540:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695986.230541:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695986.230608:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695986.230613:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695986.230616:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695986.614410:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695986.614415:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695986.614417:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695986.614421:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695986.614439:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695986.614442:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695986.614443:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695986.614445:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695986.614448:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695987.254518:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695987.254526:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695987.254531:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695987.254554:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695987.254558:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695987.254559:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695987.254563:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695987.254564:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695987.254566:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695987.254567:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695987.254570:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695987.254571:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695987.254674:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695987.254682:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695987.254687:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695987.638473:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695987.638481:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695987.638484:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695987.638490:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695987.638493:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695987.638496:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695987.638497:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695987.638500:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695987.638502:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695988.278506:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695988.278516:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695988.278525:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695988.278527:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695988.278532:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695988.278533:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695988.278538:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695988.278539:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695988.278544:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695988.278545:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695988.278549:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695988.278551:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695988.278664:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695988.278671:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695988.278676:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695988.662543:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695988.662553:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695988.662556:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695988.662563:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695988.662568:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695988.662574:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695988.662578:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695988.662582:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695988.662588:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695989.302476:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695989.302481:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695989.302485:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695989.302486:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695989.302489:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695989.302490:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695989.302492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695989.302493:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695989.302496:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695989.302497:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695989.302498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695989.302499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695989.302577:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695989.302584:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695989.302587:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787695989.622642:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695989.622653:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787695989.622656:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695989.622659:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695989.622663:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695989.622670:0:47074:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3877 00000400:00000200:2.0:1787695989.622679:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695989.622686:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695989.622689:0:47074:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695989.622692:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695989.622697:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695989.622699:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695989.622702:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695989.622706:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695989.622708:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:2.0:1787695989.622711:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695989.622715:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:2.0:1787695989.622718:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:2.0:1787695989.622720:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:2.0:1787695989.622723:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695989.622725:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695989.622728:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695989.622730:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695989.622734:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:2.0:1787695989.622737:0:47074:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695989.622740:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695989.622744:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695989.622758:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695989.622761:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695989.622764:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787695989.622767:0:47074:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695989.622775:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465af100. 00000020:00000040:2.0:1787695989.622779:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:2.0:1787695989.622783:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695989.622785:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695989.622788:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695989.622791:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695989.622796:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695989.622804:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445660800. 00000100:00000001:2.0:1787695989.622812:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695989.622823:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465af100 x1874531139557248/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695989.622834:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695989.622839:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695989.622844:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000094a5074e] to pc [ptlrpcd_rcv+-1] req@ffff8b14465af100 x1874531139557248/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695989.622870:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695989.622873:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695989.622876:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787695989.622878:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695989.622879:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695989.622882:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695989.622885:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3877 00000400:00000200:2.0:1787695989.622890:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695989.622893:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695989.622896:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695989.622899:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695989.622901:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695989.622911:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695989.622913:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695989.622914:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695989.622915:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:2.0:1787695989.622917:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695989.622919:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:2.0:1787695989.622920:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:2.0:1787695989.622922:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:2.0:1787695989.622924:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695989.622925:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695989.622926:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695989.622927:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695989.622929:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:2.0:1787695989.622931:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695989.622932:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695989.622933:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695989.622937:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695989.622939:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695989.622940:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695989.622943:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787695989.622945:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695989.622946:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695989.622949:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ac380. 00000020:00000040:2.0:1787695989.622951:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:2.0:1787695989.622953:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695989.622954:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695989.622956:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695989.622958:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695989.622960:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695989.622964:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445662000. 00000100:00000001:2.0:1787695989.622969:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695989.622974:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ac380 x1874531139557376/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695989.622987:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695989.622989:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695989.622994:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001e778d47] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ac380 x1874531139557376/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695989.623002:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695989.623005:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695989.623008:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787695989.623010:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695989.623011:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695989.623014:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695989.623016:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3877 00000400:00000200:2.0:1787695989.623021:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695989.623024:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695989.623026:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695989.623029:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695989.623031:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695989.623032:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695989.623034:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695989.623037:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695989.623038:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:2.0:1787695989.623039:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695989.623041:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:2.0:1787695989.623044:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:2.0:1787695989.623045:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:2.0:1787695989.623048:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695989.623050:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695989.623051:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695989.623053:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695989.623061:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:2.0:1787695989.623064:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695989.623066:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695989.623068:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695989.623071:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695989.623074:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695989.623076:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695989.623078:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787695989.623081:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695989.623082:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695989.623091:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ac000. 00000020:00000040:2.0:1787695989.623093:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:2.0:1787695989.623097:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695989.623098:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695989.623100:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695989.623101:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695989.623103:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695989.623107:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445660400. 00000100:00000001:2.0:1787695989.623110:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695989.623116:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ac000 x1874531139557504/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695989.623125:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695989.623127:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695989.623132:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a231b497] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ac000 x1874531139557504/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695989.623140:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695989.623142:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695989.623146:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787695989.623148:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695989.623150:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695989.623153:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695989.623162:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3877 00000400:00000200:2.0:1787695989.623167:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695989.623181:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695989.623184:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695989.623188:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695989.623190:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695989.623192:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695989.623195:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695989.623197:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695989.623198:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:2.0:1787695989.623200:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787695989.623202:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:2.0:1787695989.623204:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:2.0:1787695989.623207:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:2.0:1787695989.623210:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695989.623219:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695989.623222:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695989.623224:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695989.623228:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:2.0:1787695989.623231:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695989.623232:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695989.623234:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695989.623237:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695989.623238:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695989.623240:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695989.623242:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787695989.623243:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695989.623245:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695989.623248:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465adf80. 00000020:00000040:2.0:1787695989.623249:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:2.0:1787695989.623251:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695989.623256:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695989.623258:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695989.623259:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695989.623260:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695989.623263:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663400. 00000100:00000001:2.0:1787695989.623266:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695989.623269:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465adf80 x1874531139557632/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695989.623275:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695989.623277:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695989.623280:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ee5f8f2] to pc [ptlrpcd_rcv+-1] req@ffff8b14465adf80 x1874531139557632/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695989.623286:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787695989.623288:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695989.623291:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1787695989.623292:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695989.623293:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787695989.623296:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695989.623305:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3877 00000400:00000200:2.0:1787695989.623311:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695989.623314:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695989.623317:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695989.623321:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1787695989.623323:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695989.623329:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695989.623330:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1787695989.623332:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695989.623336:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787695989.623339:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787695989.623345:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695989.623348:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000100:00000040:1.0:1787695989.623350:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465af100 x1874531139557248/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787695989.623351:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695989.623418:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695989.623420:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695989.623424:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695989.623427:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695989.623430:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695989.623433:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139557248:192.168.201.140@tcp:250:kworker.0 00000100:00000001:1.0:1787695989.623438:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695989.623443:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695989.623455:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695989.623459:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695989.623466:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695989.623469:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695989.623478:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695989.623484:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695989.623494:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695989.623505:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139557248, portal 25 00000100:00000001:1.0:1787695989.623507:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695989.623509:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928812800 : -128555780738816 : ffff8b14465af100) 00000100:00000040:1.0:1787695989.623515:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465af100 x1874531139557248/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696014 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.623522:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695989.623524:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695989.623527:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695989.623531:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139557248, offset 0 00000400:00000200:1.0:1787695989.623537:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695989.623547:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695989.623554:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695989.623562:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695989.623567:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695989.623573:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695989.623576:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695989.623590:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17163:17163:256:4294967295] 192.168.201.140@tcp LPNI seq info [17162:17162:8:4294967295] 00000400:00000200:1.0:1787695989.623596:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695989.623623:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695989.623631:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695989.623637:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695989.623645:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695989.623649:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1787695989.623655:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695989.623659:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1787695989.623663:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695989.623666:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695989.623672:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695989.623677:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695989.623683:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695989.623685:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695989.623690:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695989.623694:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465af100 x1874531139557248/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696014 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.623705:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695989.623710:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465af100 x1874531139557248/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696014 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.623718:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.623720:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695989.623728:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695989.623734:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695989.623738:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298561336 00000100:00000001:1.0:1787695989.623748:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298556217 00000100:00000001:1.0:1787695989.623754:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298561336 00000100:00000001:1.0:1787695989.623759:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298556217 00000100:00000001:1.0:1787695989.623761:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695989.623764:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.623771:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695989.623776:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695989/real 1787695989] req@ffff8b14465af100 x1874531139557248/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695989.623785:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695989.623786:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695989.623788:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695989.623790:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695989.623792:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695989.623794:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695989.623797:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465af100 x1874531139557248/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695989.623803:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465af100 x1874531139557248/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.623808:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695989.623809:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.623811:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695989.623812:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695989.623815:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465af100 x1874531139557248/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.623821:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695989.623824:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465af100 x1874531139557248/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.623830:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695989.623832:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.623833:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695989.623834:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.623836:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695989.623838:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695989.623840:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695989.623843:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695989.623849:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695989.623853:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465af100 x1874531139557248/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695989.623868:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465af100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139557248:192.168.201.140@tcp:250:kworker.0 00000100:00000001:1.0:1787695989.623873:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695989.623875:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ac380 x1874531139557376/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.623880:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695989.623882:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695989.623884:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695989.623886:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695989.623888:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695989.623891:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139557376:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695989.623895:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695989.623898:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695989.623899:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695989.623901:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695989.623905:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695989.623907:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695989.623912:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695989.623916:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695989.623920:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695989.623924:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139557376, portal 10 00000100:00000001:1.0:1787695989.623926:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695989.623928:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928801152 : -128555780750464 : ffff8b14465ac380) 00000100:00000040:1.0:1787695989.623936:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ac380 x1874531139557376/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696014 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.623943:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695989.623945:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695989.623947:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695989.623954:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139557376, offset 0 00000400:00000200:1.0:1787695989.623957:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695989.623960:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695989.623963:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695989.623966:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695989.623968:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695989.623970:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695989.623971:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695989.623974:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17164:17164:256:4294967295] 192.168.201.140@tcp LPNI seq info [17163:17163:8:4294967295] 00000400:00000200:1.0:1787695989.623977:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695989.623984:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695989.623989:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695989.623993:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695989.623997:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695989.624000:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1787695989.624004:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695989.624007:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1787695989.624012:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695989.624014:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695989.624018:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695989.624031:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695989.624035:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695989.624037:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695989.624041:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695989.624045:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ac380 x1874531139557376/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696014 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.624051:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695989.624053:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ac380 x1874531139557376/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696014 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.624059:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.624060:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695989.624063:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695989.624071:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000400:00000200:1.0:1787695989.624075:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695989.624078:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298551237 00000100:00000001:1.0:1787695989.624081:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298561336 00000100:00000001:1.0:1787695989.624083:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298551237 00000100:00000001:1.0:1787695989.624085:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298561336 00000100:00000001:1.0:1787695989.624088:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695989.624090:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.624092:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695989.624096:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695989/real 1787695989] req@ffff8b14465ac380 x1874531139557376/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695989.624116:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695989.624118:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695989.624120:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695989.624123:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695989.624125:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695989.624128:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695989.624131:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ac380 x1874531139557376/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695989.624137:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ac380 x1874531139557376/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.624142:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695989.624143:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.624144:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695989.624145:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695989.624148:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ac380 x1874531139557376/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.624152:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695989.624155:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ac380 x1874531139557376/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.624165:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695989.624167:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.624168:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695989.624169:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.624170:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695989.624171:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695989.624173:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695989.624175:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695989.624177:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695989.624180:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ac380 x1874531139557376/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695989.624192:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139557376:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695989.624197:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695989.624201:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ac000 x1874531139557504/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.624208:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695989.624210:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695989.624212:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695989.624215:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695989.624218:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695989.624221:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139557504:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695989.624225:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695989.624228:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695989.624230:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695989.624232:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695989.624236:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695989.624239:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695989.624248:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695989.624251:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695989.624255:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695989.624259:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139557504, portal 10 00000100:00000001:1.0:1787695989.624261:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695989.624263:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928800256 : -128555780751360 : ffff8b14465ac000) 00000100:00000040:1.0:1787695989.624270:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ac000 x1874531139557504/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696014 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.624278:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695989.624281:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695989.624284:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695989.624286:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139557504, offset 0 00000400:00000200:1.0:1787695989.624291:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695989.624296:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695989.624299:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695989.624301:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695989.624304:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695989.624305:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695989.624307:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695989.624310:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17165:17165:256:4294967295] 192.168.201.140@tcp LPNI seq info [17164:17164:8:4294967295] 00000400:00000200:1.0:1787695989.624313:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695989.624318:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695989.624321:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695989.624324:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695989.624327:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695989.624329:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1787695989.624332:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695989.624334:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1787695989.624337:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695989.624339:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695989.624341:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695989.624344:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695989.624346:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695989.624353:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695989.624356:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695989.624359:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ac000 x1874531139557504/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696014 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.624384:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695989.624386:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ac000 x1874531139557504/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696014 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.624391:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.624392:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695989.624398:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695989.624401:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787695989.624404:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695989.624406:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298550085 00000100:00000001:1.0:1787695989.624408:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298561336 00000100:00000001:1.0:1787695989.624410:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298550085 00000100:00000001:1.0:1787695989.624412:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298561336 00000100:00000001:1.0:1787695989.624413:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695989.624415:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.624416:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695989.624418:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695989/real 1787695989] req@ffff8b14465ac000 x1874531139557504/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695989.624423:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695989.624424:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695989.624425:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695989.624427:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695989.624428:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695989.624429:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695989.624432:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ac000 x1874531139557504/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695989.624437:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ac000 x1874531139557504/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.624446:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695989.624448:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.624450:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695989.624451:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695989.624456:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ac000 x1874531139557504/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.624463:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695989.624467:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ac000 x1874531139557504/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.624475:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695989.624477:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.624480:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695989.624481:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.624483:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695989.624485:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695989.624488:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695989.624492:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695989.624494:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695989.624499:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ac000 x1874531139557504/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695989.624507:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139557504:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695989.624511:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695989.624513:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465adf80 x1874531139557632/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.624518:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695989.624520:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695989.624526:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695989.624527:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695989.624529:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695989.624531:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139557632:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695989.624534:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695989.624536:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695989.624537:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695989.624538:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695989.624541:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695989.624542:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695989.624545:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695989.624547:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695989.624550:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695989.624553:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139557632, portal 4 00000100:00000001:1.0:1787695989.624555:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695989.624556:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928808320 : -128555780743296 : ffff8b14465adf80) 00000100:00000040:1.0:1787695989.624562:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465adf80 x1874531139557632/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696014 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.624569:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695989.624572:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695989.624574:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695989.624577:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139557632, offset 0 00000400:00000200:1.0:1787695989.624582:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695989.624587:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695989.624591:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695989.624594:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695989.624597:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695989.624599:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695989.624613:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695989.624616:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17166:17166:256:4294967295] 192.168.201.140@tcp LPNI seq info [17165:17165:8:4294967295] 00000400:00000200:1.0:1787695989.624619:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695989.624625:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695989.624633:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695989.624636:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695989.624639:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695989.624642:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1787695989.624646:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695989.624652:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1787695989.624656:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695989.624657:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695989.624660:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695989.624662:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695989.624665:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695989.624667:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695989.624669:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695989.624673:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465adf80 x1874531139557632/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696014 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1787695989.624677:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:2.0:1787695989.624688:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000001:1.0:1787695989.624693:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695989.624702:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465adf80 x1874531139557632/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696014 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787695989.624702:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000001:1.0:1787695989.624719:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695989.624720:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695989.624729:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695989.624729:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695989.624739:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695989.624742:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695989.624742:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695989.624751:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00000040:1.0:1787695989.624755:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000100:00080000:2.0:1787695989.624756:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695989.624765:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695989.624769:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:2.0:1787695989.624769:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787695989.624776:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298550117 02000000:00000001:2.0:1787695989.624778:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695989.624782:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298561336 02000000:00000001:2.0:1787695989.624784:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.624791:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298550117 00000100:00000001:2.0:1787695989.624792:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695989.624798:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298561336 00000008:00100000:2.0:1787695989.624798:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787695989.624803:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695989.624811:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787695989.624815:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.624817:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787695989.624825:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695989.624832:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695989/real 1787695989] req@ffff8b14465adf80 x1874531139557632/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1787695989.624834:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465aea00. 00000020:00000040:2.0:1787695989.624842:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:2.0:1787695989.624853:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787695989.624857:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787695989.624863:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1787695989.624863:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:1.0:1787695989.624869:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1787695989.624870:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000010:1.0:1787695989.624881:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 02000000:00000001:2.0:1787695989.624882:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695989.624890:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661000. 00000400:00000200:1.0:1787695989.624893:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000100:00000001:2.0:1787695989.624896:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695989.624898:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00100000:2.0:1787695989.624905:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465aea00 x1874531139557760/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695989.624906:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695989.624920:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465adf80 x1874531139557632/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695989.624920:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695989.624929:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695989.624946:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a41ef0cb] to pc [ptlrpcd_rcv+-1] req@ffff8b14465aea00 x1874531139557760/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695989.624953:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465adf80 x1874531139557632/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695989.624972:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.624974:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695989.624976:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787695989.624986:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695989.624987:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695989.624992:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787695989.624993:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787695989.624998:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695989.625000:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787695989.625004:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465adf80 x1874531139557632/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695989.625005:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787695989.625011:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3877 00000400:00000200:2.0:1787695989.625022:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695989.625033:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1787695989.625034:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787695989.625045:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695989.625051:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695989.625052:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465adf80 x1874531139557632/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695989.625059:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695989.625075:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:2.0:1787695989.625077:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695989.625082:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695989.625084:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787695989.625092:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1787695989.625092:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787695989.625102:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1787695989.625102:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:2.0:1787695989.625110:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787695989.625112:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000020:00000040:2.0:1787695989.625113:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000100:00000001:1.0:1787695989.625117:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1787695989.625118:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00080000:1.0:1787695989.625129:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000020:00000001:2.0:1787695989.625131:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000040:2.0:1787695989.625146:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00080000:1.0:1787695989.625147:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695989.625159:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1787695989.625159:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695989.625165:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695989.625167:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695989.625172:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:2.0:1787695989.625178:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695989.625180:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695989.625186:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1787695989.625188:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465adf80 x1874531139557632/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695989.625191:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695989.625195:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695989.625200:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695989.625210:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787695989.625211:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465adf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139557632:192.168.201.140@tcp:8:kworker.0 00000008:00000001:2.0:1787695989.625215:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695989.625227:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.625230:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:2.0:1787695989.625231:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465aed80. 00000100:00000001:1.0:1787695989.625236:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000040:2.0:1787695989.625236:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:2.0:1787695989.625243:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000040:1.0:1787695989.625246:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465af100 x1874531139557248/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695989.625246:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695989.625252:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695989.625254:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695989.625258:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695989.625267:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663000. 00000100:00000001:2.0:1787695989.625274:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.625277:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695989.625282:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:2.0:1787695989.625285:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465aed80 x1874531139557888/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:1.0:1787695989.625287:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695989.625299:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695989.625302:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695989.625303:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:1.0:1787695989.625306:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695989.625311:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445660800. 02000000:00000001:1.0:1787695989.625318:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695989.625320:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695989.625325:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465af100. 00000100:00000001:2.0:1787695989.625326:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695989.625329:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695989.625332:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695989.625332:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.625341:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695989.625350:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ac380 x1874531139557376/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787695989.625352:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000bed91c58] to pc [ptlrpcd_rcv+-1] req@ffff8b14465aed80 x1874531139557888/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.625385:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695989.625387:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695989.625389:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695989.625392:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695989.625396:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695989.625399:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695989.625406:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695989.625411:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445662000. 02000000:00000001:1.0:1787695989.625420:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1787695989.625423:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695989.625424:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:2.0:1787695989.625428:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787695989.625431:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000010:1.0:1787695989.625433:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ac380. 00000100:00000001:2.0:1787695989.625434:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787695989.625444:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695989.625447:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695989.625448:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695989.625454:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787695989.625458:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3877 00000100:00000001:1.0:1787695989.625462:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1787695989.625466:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695989.625473:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000040:1.0:1787695989.625474:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ac000 x1874531139557504/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787695989.625475:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787695989.625481:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695989.625490:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787695989.625496:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695989.625505:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1787695989.625506:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:1.0:1787695989.625508:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695989.625513:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 00000020:00000001:2.0:1787695989.625514:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787695989.625520:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 02000000:00000001:1.0:1787695989.625523:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695989.625524:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787695989.625526:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695989.625528:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000040:1.0:1787695989.625529:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695989.625544:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000020:00000001:2.0:1787695989.625544:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 02000000:00000010:1.0:1787695989.625553:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445660400. 00000020:00000001:2.0:1787695989.625554:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:2.0:1787695989.625565:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:1.0:1787695989.625566:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1787695989.625569:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787695989.625570:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:2.0:1787695989.625573:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000010:1.0:1787695989.625576:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ac000. 00000100:00000001:2.0:1787695989.625577:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787695989.625586:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00000001:1.0:1787695989.625587:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00080000:2.0:1787695989.625590:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695989.625593:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695989.625593:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.625615:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1787695989.625621:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1787695989.625624:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465adf80 x1874531139557632/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787695989.625627:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695989.625638:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695989.625645:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787695989.625651:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787695989.625653:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695989.625658:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000008:00000001:2.0:1787695989.625658:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695989.625660:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695989.625667:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787695989.625667:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787695989.625671:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695989.625675:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695989.625678:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695989.625681:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663400. 00000100:00000010:2.0:1787695989.625681:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ae300. 02000000:00000001:1.0:1787695989.625684:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695989.625689:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695989.625691:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465adf80. 00000020:00000040:2.0:1787695989.625692:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000001:1.0:1787695989.625694:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695989.625698:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787695989.625700:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787695989.625702:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787695989.625705:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695989.625707:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787695989.625709:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695989.625713:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787695989.625722:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661400. 00000100:00000001:1.0:1787695989.625724:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695989.625725:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.625732:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695989.625738:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ae300 x1874531139558016/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695989.625740:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695989.625744:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695989.625748:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695989.625757:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465aea00 x1874531139557760/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.625786:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695989.625790:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695989.625801:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695989.625804:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695989.625805:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787695989.625807:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695989.625813:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139557760:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787695989.625814:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.625826:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695989.625830:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695989.625832:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695989.625834:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004515329a] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ae300 x1874531139558016/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695989.625835:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695989.625841:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695989.625847:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695989.625866:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695989.625875:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000100:00000001:2.0:1787695989.625878:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695989.625882:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000040:2.0:1787695989.625885:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000200:1.0:1787695989.625890:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139557760, portal 4 00000100:00000040:2.0:1787695989.625892:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:1.0:1787695989.625895:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695989.625900:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928811008 : -128555780740608 : ffff8b14465aea00) 00000100:00000040:2.0:1787695989.625901:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:2.0:1787695989.625910:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1787695989.625916:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3887) 00000100:00000040:1.0:1787695989.625918:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465aea00 x1874531139557760/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696014 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.625928:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695989.625930:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695989.625933:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695989.625936:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139557760, offset 0 00000400:00000200:1.0:1787695989.625939:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695989.625945:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695989.625950:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695989.625960:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695989.625964:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695989.625966:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695989.625969:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695989.625972:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17167:17167:256:4294967295] 192.168.201.140@tcp LPNI seq info [17166:17166:8:4294967295] 00000400:00000200:1.0:1787695989.625978:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695989.625986:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695989.625990:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695989.625994:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695989.625999:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695989.626002:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1787695989.626008:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695989.626011:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1787695989.626015:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695989.626017:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695989.626020:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695989.626023:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695989.626026:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695989.626028:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695989.626031:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695989.626035:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465aea00 x1874531139557760/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696014 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626043:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695989.626046:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465aea00 x1874531139557760/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696014 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626053:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.626055:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695989.626058:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695989.626062:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695989.626066:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695989.626068:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298549790 00000100:00000001:1.0:1787695989.626071:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298561336 00000100:00000001:1.0:1787695989.626079:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298549790 00000100:00000001:1.0:1787695989.626081:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298561336 00000100:00000001:1.0:1787695989.626084:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695989.626087:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.626089:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695989.626095:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695989/real 1787695989] req@ffff8b14465aea00 x1874531139557760/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695989.626102:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695989.626103:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695989.626105:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695989.626107:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695989.626108:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695989.626110:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695989.626112:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465aea00 x1874531139557760/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695989.626118:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465aea00 x1874531139557760/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626122:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695989.626123:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.626125:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695989.626126:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695989.626128:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465aea00 x1874531139557760/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626133:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695989.626137:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465aea00 x1874531139557760/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626144:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695989.626147:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.626149:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695989.626155:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.626157:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695989.626159:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695989.626162:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695989.626167:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695989.626169:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695989.626174:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465aea00 x1874531139557760/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695989.626183:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465aea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139557760:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695989.626188:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695989.626194:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465aed80 x1874531139557888/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626201:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695989.626203:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695989.626205:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695989.626207:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695989.626209:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695989.626213:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139557888:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695989.626217:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695989.626219:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695989.626221:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695989.626223:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695989.626227:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695989.626229:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695989.626233:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695989.626236:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695989.626240:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695989.626243:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139557888, portal 10 00000100:00000001:1.0:1787695989.626245:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695989.626247:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928811904 : -128555780739712 : ffff8b14465aed80) 00000100:00000040:1.0:1787695989.626259:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465aed80 x1874531139557888/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696014 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626266:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695989.626268:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695989.626270:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695989.626272:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139557888, offset 0 00000400:00000200:1.0:1787695989.626277:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695989.626282:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695989.626286:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695989.626289:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695989.626292:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695989.626294:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695989.626297:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695989.626300:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17168:17168:256:4294967295] 192.168.201.140@tcp LPNI seq info [17167:17167:8:4294967295] 00000400:00000200:1.0:1787695989.626305:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695989.626312:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695989.626317:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695989.626321:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695989.626326:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695989.626328:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1787695989.626331:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695989.626333:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1787695989.626336:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695989.626338:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695989.626340:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695989.626343:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695989.626345:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695989.626347:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695989.626349:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695989.626352:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465aed80 x1874531139557888/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696014 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626358:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695989.626388:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465aed80 x1874531139557888/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696014 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626393:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.626395:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695989.626401:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695989.626406:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695989.626411:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695989.626414:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298549977 00000100:00000001:1.0:1787695989.626417:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298561336 00000100:00000001:1.0:1787695989.626420:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298549977 00000100:00000001:1.0:1787695989.626423:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298561336 00000100:00000001:1.0:1787695989.626425:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695989.626428:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.626430:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695989.626436:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695989/real 1787695989] req@ffff8b14465aed80 x1874531139557888/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695989.626447:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695989.626450:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695989.626452:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695989.626456:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695989.626457:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695989.626460:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695989.626464:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465aed80 x1874531139557888/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695989.626473:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465aed80 x1874531139557888/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626480:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695989.626482:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.626484:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695989.626486:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695989.626498:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465aed80 x1874531139557888/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626506:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695989.626511:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465aed80 x1874531139557888/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626517:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695989.626519:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.626520:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695989.626521:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.626522:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695989.626524:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695989.626525:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695989.626528:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695989.626530:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695989.626533:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465aed80 x1874531139557888/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695989.626537:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465aed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139557888:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695989.626540:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.626541:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695989.626543:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465aea00 x1874531139557760/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626548:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695989.626549:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695989.626551:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695989.626553:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695989.626554:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695989.626555:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695989.626557:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695989.626563:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661000. 02000000:00000001:1.0:1787695989.626567:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695989.626569:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695989.626571:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465aea00. 00000100:00000001:1.0:1787695989.626573:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695989.626575:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.626577:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695989.626581:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465aed80 x1874531139557888/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626588:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695989.626590:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695989.626592:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695989.626594:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695989.626595:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695989.626597:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695989.626599:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695989.626600:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663000. 02000000:00000001:1.0:1787695989.626621:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695989.626623:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695989.626624:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465aed80. 00000100:00000001:1.0:1787695989.626627:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695989.626629:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.626631:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.626635:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695989.626637:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.626639:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695989.626640:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695989.626643:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695989.626646:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ae300 x1874531139558016/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626652:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695989.626653:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695989.626655:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695989.626657:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695989.626658:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695989.626660:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139558016:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695989.626667:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695989.626669:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695989.626671:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695989.626672:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695989.626674:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695989.626675:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695989.626678:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695989.626680:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695989.626683:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695989.626685:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139558016, portal 10 00000100:00000001:1.0:1787695989.626687:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695989.626688:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928809216 : -128555780742400 : ffff8b14465ae300) 00000100:00000040:1.0:1787695989.626693:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ae300 x1874531139558016/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696014 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626698:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695989.626700:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695989.626702:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695989.626704:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139558016, offset 0 00000400:00000200:1.0:1787695989.626709:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695989.626713:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695989.626717:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695989.626721:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695989.626723:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695989.626726:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695989.626729:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695989.626732:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17169:17169:256:4294967295] 192.168.201.140@tcp LPNI seq info [17168:17168:8:4294967295] 00000400:00000200:1.0:1787695989.626737:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695989.626744:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695989.626749:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695989.626754:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695989.626757:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695989.626764:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000200:1.1:1787695989.626768:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695989.626771:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 12 secs later) 00000800:00000100:1.0:1787695989.626776:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695989.626778:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695989.626781:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695989.626785:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695989.626789:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695989.626791:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695989.626793:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695989.626798:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ae300 x1874531139558016/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696014 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626807:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695989.626810:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ae300 x1874531139558016/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696014 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626818:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.626819:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695989.626823:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695989.626826:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695989.626830:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695989.626833:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298549955 00000100:00000001:1.0:1787695989.626836:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298561336 00000100:00000001:1.0:1787695989.626838:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298549955 00000100:00000001:1.0:1787695989.626840:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298561336 00000100:00000001:1.0:1787695989.626843:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695989.626845:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.626847:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695989.626852:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695989/real 1787695989] req@ffff8b14465ae300 x1874531139558016/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695989.626861:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695989.626863:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695989.626870:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695989.626873:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695989.626875:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695989.626878:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695989.626882:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ae300 x1874531139558016/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695989.626888:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ae300 x1874531139558016/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626892:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695989.626893:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.626894:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695989.626895:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695989.626898:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ae300 x1874531139558016/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626903:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695989.626905:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ae300 x1874531139558016/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626909:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695989.626911:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.626912:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695989.626913:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.626914:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695989.626915:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695989.626916:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695989.626918:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695989.626920:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695989.626922:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ae300 x1874531139558016/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695989.626927:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139558016:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695989.626933:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.626934:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695989.626936:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ae300 x1874531139558016/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696014 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695989.626941:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695989.626942:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695989.626943:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695989.626945:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695989.626946:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695989.626947:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695989.626948:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695989.626949:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661400. 02000000:00000001:1.0:1787695989.626951:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695989.626952:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695989.626953:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ae300. 00000100:00000001:1.0:1787695989.626955:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695989.626956:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.626957:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695989.626960:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695989.626961:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695989.626962:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695989.626963:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695989.686508:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695989.686519:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695989.686521:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695989.686527:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695989.686532:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695989.686537:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695989.686540:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695989.686543:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695989.686547:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695990.326510:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695990.326520:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695990.326528:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695990.326530:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695990.326535:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695990.326536:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695990.326541:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695990.326543:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695990.326547:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695990.326548:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695990.326553:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695990.326554:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695990.326645:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695990.326653:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695990.326657:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695990.710489:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695990.710498:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695990.710500:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695990.710524:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695990.710528:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695990.710532:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695990.710535:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695990.710538:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695990.710541:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695991.350525:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695991.350533:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695991.350561:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695991.350562:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695991.350566:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695991.350567:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695991.350570:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695991.350595:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695991.350599:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695991.350602:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695991.350605:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695991.350611:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695991.350613:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695991.350615:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695991.350618:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695991.350620:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695991.350622:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695991.350623:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695991.350627:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695991.350628:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695991.350696:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695991.350702:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695991.350705:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695991.734515:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695991.734523:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695991.734525:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695991.734531:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695991.734534:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695991.734537:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695991.734539:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695991.734540:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695991.734542:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695992.374569:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695992.374579:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695992.374587:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695992.374588:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695992.374593:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695992.374594:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695992.374599:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695992.374600:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695992.374604:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695992.374606:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695992.374610:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695992.374611:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695992.374848:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695992.374854:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695992.374859:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695992.758485:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695992.758493:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695992.758495:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695992.758501:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695992.758505:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695992.758509:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695992.758511:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695992.758514:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695992.758516:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695993.398468:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695993.398473:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695993.398477:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695993.398478:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695993.398480:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695993.398481:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695993.398484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695993.398485:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695993.398505:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695993.398506:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695993.398509:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695993.398510:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695993.398575:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695993.398580:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695993.398584:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695993.782459:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695993.782466:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695993.782467:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695993.782471:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695993.782474:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695993.782478:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695993.782480:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695993.782482:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695993.782484:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695994.422514:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695994.422682:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695994.422686:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695994.422691:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695994.422698:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695994.422701:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695994.422704:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695994.422707:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695994.422714:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695994.422716:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695994.422718:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695994.422720:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695994.422722:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695994.422723:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695994.422725:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695994.422727:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695994.422733:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695994.422735:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695994.422740:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695994.422742:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695994.422746:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695994.422747:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695994.422752:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695994.422753:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695994.422823:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695994.422851:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695994.422855:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787695994.743399:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695994.743422:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787695994.743424:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695994.743427:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695994.743430:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695994.743436:0:47074:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3882 00000400:00000200:3.0:1787695994.743444:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695994.743449:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695994.743452:0:47074:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695994.743454:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.743456:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695994.743459:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.743461:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695994.743464:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695994.743466:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:3.0:1787695994.743468:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695994.743472:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787695994.743475:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787695994.743477:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787695994.743479:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.743481:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695994.743483:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.743484:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695994.743487:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787695994.743492:0:47074:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695994.743494:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695994.743497:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695994.743501:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695994.743503:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695994.743505:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787695994.743507:0:47074:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695994.743512:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5880. 00000020:00000040:3.0:1787695994.743532:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787695994.743537:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695994.743538:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695994.743545:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695994.743548:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695994.743553:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695994.743561:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675400. 00000100:00000001:3.0:1787695994.743568:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695994.743578:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5880 x1874531139558144/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787695994.743589:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695994.743593:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695994.743598:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000827e7dbe] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5880 x1874531139558144/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695994.743620:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695994.743622:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695994.743625:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695994.743626:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695994.743627:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695994.743629:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787695994.743631:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3882 00000400:00000200:3.0:1787695994.743634:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695994.743636:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695994.743638:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695994.743640:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.743642:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695994.743644:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.743646:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695994.743647:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695994.743648:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:3.0:1787695994.743650:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695994.743658:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:3.0:1787695994.743660:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:3.0:1787695994.743662:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:3.0:1787695994.743665:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695994.743670:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695994.743671:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695994.743678:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.743683:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695994.743684:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00000001:1.0:1787695994.743687:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1787695994.743687:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695994.743695:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695994.743696:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787695994.743699:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695994.743708:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695994.743716:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695994.743717:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5880 x1874531139558144/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695994.743720:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695994.743725:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695994.743743:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695994.743757:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695994.743758:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695994.743766:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695994.743769:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6680. 00000020:00000040:3.0:1787695994.743774:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:1.0:1787695994.743775:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787695994.743779:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695994.743782:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695994.743783:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695994.743786:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695994.743787:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695994.743789:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695994.743794:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 00000100:00000001:3.0:1787695994.743798:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695994.743808:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695994.743809:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6680 x1874531139558272/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787695994.743821:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139558144:192.168.201.140@tcp:250:kworker.0 00000100:00000001:1.0:1787695994.743841:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787695994.743849:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787695994.743850:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787695994.743857:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695994.743859:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695994.743868:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f6d2abf9] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6680 x1874531139558272/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695994.743869:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695994.743883:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695994.743890:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695994.743909:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695994.743911:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:3.0:1787695994.743913:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695994.743918:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000400:00000010:1.0:1787695994.743920:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000100:00000001:3.0:1787695994.743921:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695994.743927:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695994.743931:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000400:00000010:1.0:1787695994.743935:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00080000:3.0:1787695994.743936:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3882 00000400:00000200:3.0:1787695994.743948:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000200:1.0:1787695994.743955:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139558144, portal 25 00000400:00000200:3.0:1787695994.743955:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787695994.743970:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787695994.743970:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695994.743981:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150016 : -128555779401600 : ffff8b14466f5880) 00000100:00000040:3.0:1787695994.743983:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.743995:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695994.744001:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000040:1.0:1787695994.744006:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5880 x1874531139558144/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696019 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695994.744007:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695994.744014:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695994.744018:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:3.0:1787695994.744024:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695994.744030:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:3.0:1787695994.744038:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:3.0:1787695994.744050:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000001:1.0:1787695994.744051:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1787695994.744056:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.744059:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695994.744062:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695994.744066:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000010:1.0:1787695994.744069:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000001:3.0:1787695994.744070:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695994.744080:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00000200:1.0:1787695994.744082:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139558144, offset 0 00000100:00080000:3.0:1787695994.744085:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695994.744090:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695994.744098:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1787695994.744104:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695994.744105:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695994.744111:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695994.744112:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695994.744114:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1787695994.744122:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000008:00000001:3.0:1787695994.744125:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695994.744128:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695994.744131:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4a80. 00000400:00000200:1.0:1787695994.744134:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000020:00000040:3.0:1787695994.744136:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:3.0:1787695994.744148:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695994.744150:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695994.744152:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1787695994.744154:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695994.744157:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695994.744160:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695994.744164:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 02000000:00000010:3.0:1787695994.744167:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675800. 00000100:00000001:3.0:1787695994.744171:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695994.744173:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00100000:3.0:1787695994.744180:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4a80 x1874531139558400/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787695994.744184:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.744201:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1787695994.744202:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17170:17170:256:4294967295] 192.168.201.140@tcp LPNI seq info [17169:17169:8:4294967295] 00000100:00000001:3.0:1787695994.744205:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695994.744218:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ad0d67] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4a80 x1874531139558400/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787695994.744222:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787695994.744234:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695994.744236:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695994.744251:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000200:1.0:1787695994.744252:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:3.0:1787695994.744258:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695994.744263:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.0:1787695994.744266:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787695994.744271:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000800:00000010:1.0:1787695994.744276:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000100:00080000:3.0:1787695994.744279:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3882 00000400:00000200:3.0:1787695994.744289:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695994.744301:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000800:00000200:1.0:1787695994.744303:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00080000:3.0:1787695994.744310:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000800:00000200:1.0:1787695994.744313:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1787695994.744326:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695994.744331:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 7 secs later) 00000100:00000040:3.0:1787695994.744333:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.744340:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:1.0:1787695994.744342:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787695994.744350:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000010:1.0:1787695994.744352:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000100:00000001:3.0:1787695994.744354:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695994.744359:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695994.744360:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:3.0:1787695994.744391:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695994.744393:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:3.0:1787695994.744395:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:3.0:1787695994.744398:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:3.0:1787695994.744402:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.744405:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695994.744407:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.744409:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787695994.744413:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000400:00000200:1.0:1787695994.744416:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00080000:3.0:1787695994.744418:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695994.744426:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695994.744430:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 02000000:00000001:3.0:1787695994.744432:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695994.744441:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1787695994.744442:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 02000000:00000001:3.0:1787695994.744445:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787695994.744448:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:3.0:1787695994.744450:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787695994.744460:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787695994.744462:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000008:00000001:3.0:1787695994.744465:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695994.744468:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787695994.744475:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5880 x1874531139558144/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696019 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1787695994.744475:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7b80. 00000020:00000040:3.0:1787695994.744483:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:3.0:1787695994.744491:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695994.744498:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695994.744505:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695994.744509:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695994.744525:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695994.744534:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 00000100:00000001:1.0:1787695994.744537:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695994.744546:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5880 x1874531139558144/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696019 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695994.744547:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695994.744560:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7b80 x1874531139558528/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695994.744578:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695994.744581:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695994.744585:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787695994.744587:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695994.744595:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b87e1da] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7b80 x1874531139558528/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.744598:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695994.744611:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695994.744619:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:3.0:1787695994.744622:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695994.744626:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298566458 00000100:00080000:3.0:1787695994.744626:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1787695994.744635:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695994.744640:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695994.744645:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787695994.744649:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298561336 00000100:00080000:3.0:1787695994.744650:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3882 00000100:00000001:1.0:1787695994.744666:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298566458 00000400:00000200:3.0:1787695994.744667:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695994.744687:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298561336 00000400:00000200:3.0:1787695994.744687:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695994.744700:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787695994.744701:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695994.744709:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695994.744713:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695994.744715:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787695994.744721:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1787695994.744726:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695994/real 1787695994] req@ffff8b14466f5880 x1874531139558144/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787695994.744732:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.744743:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695994.744755:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695994.744759:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000400:00000001:1.0:1787695994.744761:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:3.0:1787695994.744763:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000001:1.0:1787695994.744765:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000040:3.0:1787695994.744767:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000400:00000010:1.0:1787695994.744771:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000020:00000001:3.0:1787695994.744771:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:3.0:1787695994.744775:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:3.0:1787695994.744778:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.744787:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695994.744789:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695994.744794:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000040:3.0:1787695994.744802:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695994.744803:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787695994.744811:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000200:1.0:1787695994.744814:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5880 x1874531139558144/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1787695994.744822:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:3.0:1787695994.744836:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000200:1.0:1787695994.744841:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5880 x1874531139558144/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695994.744844:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695994.744853:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787695994.744861:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695994.744865:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:3.0:1787695994.744865:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695994.744870:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695994.744871:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1787695994.744876:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787695994.744878:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000001:3.0:1787695994.744883:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695994.744885:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695994.744887:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787695994.744894:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7800. 00000100:00100000:1.0:1787695994.744898:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5880 x1874531139558144/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1787695994.744899:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:3.0:1787695994.744906:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695994.744909:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695994.744917:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695994.744923:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787695994.744926:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695994.744930:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1787695994.744932:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 00000100:00000001:3.0:1787695994.744938:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695994.744946:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5880 x1874531139558144/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787695994.744952:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7800 x1874531139558656/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695994.745010:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787695994.745011:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695994.745018:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695994.745021:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787695994.745026:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7c57981] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7800 x1874531139558656/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.745027:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695994.745031:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.745041:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695994.745046:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695994.745053:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787695994.745057:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695994.745062:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00080000:3.0:1787695994.745066:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787695994.745068:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:3.0:1787695994.745072:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787695994.745076:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:1.0:1787695994.745079:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5880 x1874531139558144/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787695994.745087:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787695994.745099:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:1.0:1787695994.745103:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139558144:192.168.201.140@tcp:250:kworker.0 00000100:00080000:3.0:1787695994.745106:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3882 00000400:00000200:3.0:1787695994.745117:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695994.745121:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787695994.745122:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695994.745124:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787695994.745126:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.745128:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787695994.745131:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695994.745136:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787695994.745137:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787695994.745141:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787695994.745143:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:3.0:1787695994.745147:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000040:1.0:1787695994.745148:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5880 x1874531139558144/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1787695994.745150:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000020:00000001:3.0:1787695994.745156:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000020:00000001:3.0:1787695994.745162:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000040:3.0:1787695994.745179:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695994.745180:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695994.745186:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1787695994.745187:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000010:1.0:1787695994.745193:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 00000100:00000040:3.0:1787695994.745193:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.745202:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 02000000:00000001:1.0:1787695994.745205:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000040:3.0:1787695994.745206:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000020:00000001:1.0:1787695994.745211:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000100:00080000:3.0:1787695994.745216:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000020:00000040:1.0:1787695994.745218:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000100:00000001:3.0:1787695994.745224:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787695994.745229:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787695994.745229:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:1.0:1787695994.745240:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675400. 02000000:00000001:3.0:1787695994.745240:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695994.745252:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695994.745255:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787695994.745257:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000008:00100000:3.0:1787695994.745259:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787695994.745261:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000008:00000001:3.0:1787695994.745269:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695994.745270:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5880. 00000100:00000001:1.0:1787695994.745280:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787695994.745280:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695994.745284:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.745288:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:3.0:1787695994.745295:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5c00. 00000020:00000040:3.0:1787695994.745298:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000100:00000001:1.0:1787695994.745302:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1787695994.745302:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787695994.745310:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787695994.745310:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695994.745322:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787695994.745322:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787695994.745327:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787695994.745328:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787695994.745333:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787695994.745334:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787695994.745343:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676000. 00000100:00000040:1.0:1787695994.745345:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6680 x1874531139558272/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787695994.745348:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695994.745383:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5c00 x1874531139558784/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695994.745401:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695994.745405:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695994.745406:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695994.745410:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695994.745417:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1787695994.745417:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005bfbcee1] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5c00 x1874531139558784/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695994.745424:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787695994.745432:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695994.745433:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695994.745436:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787695994.745442:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787695994.745449:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139558272:192.168.201.140@tcp:38:kworker.0 00000100:00000001:3.0:1787695994.745449:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787695994.745454:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695994.745460:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787695994.745462:0:47074:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1787695994.745465:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695994.745470:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787695994.745471:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3882 02000000:00000001:1.0:1787695994.745475:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695994.745481:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547800. 00000400:00000200:3.0:1787695994.745482:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787695994.745486:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787695994.745488:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1787695994.745496:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695994.745497:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.745506:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787695994.745511:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787695994.745512:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.745535:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000010:1.0:1787695994.745537:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000020:00000001:3.0:1787695994.745540:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000010:1.0:1787695994.745547:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695994.745551:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139558272, portal 10 00000020:00000040:3.0:1787695994.745555:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000100:00000001:1.0:1787695994.745556:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695994.745561:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930153600 : -128555779398016 : ffff8b14466f6680) 00000020:00000001:3.0:1787695994.745561:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787695994.745567:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:3.0:1787695994.745573:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:1.0:1787695994.745579:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6680 x1874531139558272/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696019 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787695994.745579:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:3.0:1787695994.745591:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.745603:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787695994.745609:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1787695994.745611:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.745620:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000040:1.0:1787695994.745621:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695994.745628:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000020:00000040:3.0:1787695994.745629:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00000200:1.0:1787695994.745635:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139558272, offset 0 00000100:00080000:3.0:1787695994.745639:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787695994.745646:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695994.745650:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787695994.745653:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1787695994.745660:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787695994.745661:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787695994.745666:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695994.745668:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:1.0:1787695994.745674:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000008:00100000:3.0:1787695994.745674:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787695994.745683:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695994.745686:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695994.745687:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000010:3.0:1787695994.745690:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6a00. 00000400:00000200:1.0:1787695994.745696:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000020:00000040:3.0:1787695994.745698:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000400:00000200:1.0:1787695994.745705:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 02000000:00000001:3.0:1787695994.745707:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787695994.745711:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695994.745712:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:3.0:1787695994.745717:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787695994.745723:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1787695994.745724:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17171:17171:256:4294967295] 192.168.201.140@tcp LPNI seq info [17170:17170:8:4294967295] 02000000:00000001:3.0:1787695994.745727:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695994.745737:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000010:3.0:1787695994.745737:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 00000100:00000001:3.0:1787695994.745746:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695994.745756:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00100000:3.0:1787695994.745762:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6a00 x1874531139558912/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.0:1787695994.745778:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695994.745787:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000100:00000001:3.0:1787695994.745789:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787695994.745795:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787695994.745797:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695994.745802:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1787695994.745816:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:3.0:1787695994.745817:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002ad47e8f] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6a00 x1874531139558912/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1787695994.745822:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1787695994.745837:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695994.745843:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695994.745854:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1787695994.745855:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787695994.745864:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787695994.745868:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00000040:3.0:1787695994.745872:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:3.0:1787695994.745875:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787695994.745888:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1787695994.745889:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695994.745895:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000040:3.0:1787695994.745899:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3892) 00000100:00000001:1.0:1787695994.745900:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695994.745909:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6680 x1874531139558272/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696019 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.745921:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695994.745926:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6680 x1874531139558272/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696019 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.745934:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695994.745936:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695994.745940:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695994.745945:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000400:00000200:1.0:1787695994.745949:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695994.745952:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298556360 00000100:00000001:1.0:1787695994.745955:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298566458 00000100:00000001:1.0:1787695994.745957:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298556360 00000100:00000001:1.0:1787695994.745960:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298566458 00000100:00000001:1.0:1787695994.745962:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695994.745964:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695994.745967:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695994.745971:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695994/real 1787695994] req@ffff8b14466f6680 x1874531139558272/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695994.745980:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695994.745981:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695994.745983:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695994.745985:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695994.745987:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695994.745990:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695994.745993:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6680 x1874531139558272/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695994.746018:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6680 x1874531139558272/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.746025:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695994.746027:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.746029:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695994.746031:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695994.746035:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6680 x1874531139558272/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.746042:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695994.746046:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6680 x1874531139558272/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.746054:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695994.746056:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.746058:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695994.746060:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.746062:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695994.746064:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695994.746067:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695994.746071:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695994.746073:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695994.746077:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6680 x1874531139558272/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695994.746085:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139558272:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695994.746090:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695994.746095:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4a80 x1874531139558400/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.746114:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695994.746116:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695994.746119:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695994.746121:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695994.746123:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695994.746126:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139558400:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695994.746130:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695994.746133:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695994.746134:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695994.746136:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695994.746140:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695994.746143:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695994.746147:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695994.746150:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695994.746154:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695994.746157:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139558400, portal 10 00000100:00000001:1.0:1787695994.746160:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695994.746161:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930146432 : -128555779405184 : ffff8b14466f4a80) 00000100:00000040:1.0:1787695994.746168:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4a80 x1874531139558400/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696019 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.746176:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695994.746178:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695994.746181:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695994.746184:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139558400, offset 0 00000400:00000200:1.0:1787695994.746189:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695994.746194:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695994.746198:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695994.746201:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695994.746204:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695994.746206:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695994.746209:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695994.746218:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17172:17172:256:4294967295] 192.168.201.140@tcp LPNI seq info [17171:17171:8:4294967295] 00000400:00000200:1.0:1787695994.746222:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695994.746229:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695994.746234:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695994.746237:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695994.746241:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695994.746244:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1787695994.746249:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695994.746251:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1787695994.746256:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695994.746258:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695994.746262:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695994.746266:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695994.746269:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695994.746271:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695994.746274:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695994.746278:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4a80 x1874531139558400/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696019 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.746286:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695994.746289:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4a80 x1874531139558400/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696019 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.746296:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695994.746298:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695994.746301:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695994.746304:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787695994.746307:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695994.746309:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298555208 00000100:00000001:1.0:1787695994.746311:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298566458 00000100:00000001:1.0:1787695994.746314:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298555208 00000100:00000001:1.0:1787695994.746316:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298566458 00000100:00000001:1.0:1787695994.746318:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695994.746324:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695994.746326:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695994.746330:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695994/real 1787695994] req@ffff8b14466f4a80 x1874531139558400/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695994.746339:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695994.746341:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695994.746343:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695994.746345:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695994.746347:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695994.746349:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695994.746352:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4a80 x1874531139558400/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695994.746362:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4a80 x1874531139558400/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.746388:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695994.746390:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.746392:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695994.746393:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695994.746397:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4a80 x1874531139558400/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.746405:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695994.746409:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4a80 x1874531139558400/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.746417:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695994.746420:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.746422:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695994.746423:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.746425:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695994.746427:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695994.746430:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695994.746439:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695994.746442:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695994.746446:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4a80 x1874531139558400/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695994.746454:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139558400:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695994.746458:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695994.746462:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7b80 x1874531139558528/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.746469:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695994.746471:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695994.746473:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695994.746475:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695994.746477:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695994.746480:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139558528:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695994.746483:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695994.746486:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695994.746488:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695994.746489:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695994.746493:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695994.746495:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695994.746499:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695994.746503:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695994.746506:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695994.746510:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139558528, portal 4 00000100:00000001:1.0:1787695994.749594:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695994.749599:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930158976 : -128555779392640 : ffff8b14466f7b80) 00000100:00000040:1.0:1787695994.749614:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7b80 x1874531139558528/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696019 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.749624:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695994.749637:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695994.749643:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695994.749649:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139558528, offset 0 00000400:00000200:1.0:1787695994.749656:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695994.749663:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695994.749671:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695994.749676:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695994.749679:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695994.749683:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695994.749685:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695994.749689:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17173:17173:256:4294967295] 192.168.201.140@tcp LPNI seq info [17172:17172:8:4294967295] 00000400:00000200:1.0:1787695994.749694:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695994.749703:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695994.749709:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695994.749713:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695994.749719:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695994.749722:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1787695994.749727:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695994.749731:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1787695994.749735:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695994.749738:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695994.749742:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695994.749747:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695994.749751:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695994.749754:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695994.749757:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695994.749762:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7b80 x1874531139558528/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696019 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.749772:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695994.749775:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7b80 x1874531139558528/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696019 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.749783:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695994.749785:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695994.749794:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695994.749798:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787695994.749802:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695994.749805:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298555242 00000100:00000001:1.0:1787695994.749808:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298566458 00000100:00000001:1.0:1787695994.749811:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298555242 00000100:00000001:1.0:1787695994.749814:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298566458 00000100:00000001:1.0:1787695994.749817:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695994.749819:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695994.749822:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695994.749826:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695994/real 1787695994] req@ffff8b14466f7b80 x1874531139558528/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695994.749836:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695994.749838:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695994.749840:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695994.749843:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695994.749845:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695994.749848:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695994.749851:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7b80 x1874531139558528/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695994.749861:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7b80 x1874531139558528/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.749868:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695994.749871:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.749873:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695994.749875:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695994.749879:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7b80 x1874531139558528/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.749886:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695994.749891:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7b80 x1874531139558528/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.749906:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695994.749909:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.749911:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695994.749912:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.749915:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695994.749917:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695994.749920:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695994.749925:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695994.749927:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695994.749931:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7b80 x1874531139558528/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695994.749941:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139558528:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695994.749946:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695994.749950:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7800 x1874531139558656/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.749958:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695994.749960:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695994.749962:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695994.749965:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695994.749967:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695994.749971:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139558656:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695994.749974:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695994.749976:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695994.749978:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695994.749980:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695994.749984:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695994.749986:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695994.749990:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695994.749997:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695994.750001:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695994.750005:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139558656, portal 4 00000100:00000001:1.0:1787695994.750007:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695994.750009:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930158080 : -128555779393536 : ffff8b14466f7800) 00000100:00000040:1.0:1787695994.750015:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7800 x1874531139558656/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696019 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.750023:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695994.750026:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695994.750028:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695994.750030:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139558656, offset 0 00000400:00000200:1.0:1787695994.750035:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695994.750039:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695994.750043:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695994.750046:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695994.750049:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695994.750052:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695994.750054:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695994.750058:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17174:17174:256:4294967295] 192.168.201.140@tcp LPNI seq info [17173:17173:8:4294967295] 00000400:00000200:1.0:1787695994.750063:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695994.750070:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695994.750076:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695994.750079:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695994.750083:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695994.750086:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1787695994.750090:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695994.750093:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1787695994.750097:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695994.750099:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695994.750102:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695994.750107:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695994.750110:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695994.750112:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695994.750121:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695994.750126:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7800 x1874531139558656/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696019 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.750134:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695994.750137:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7800 x1874531139558656/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696019 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.750145:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695994.750147:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695994.750151:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695994.750154:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787695994.750158:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695994.750160:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298554914 00000100:00000001:1.0:1787695994.750163:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298566458 00000100:00000001:1.0:1787695994.750165:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298554914 00000100:00000001:1.0:1787695994.750167:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298566458 00000100:00000001:1.0:1787695994.750170:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695994.750172:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695994.750174:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695994.750178:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695994/real 1787695994] req@ffff8b14466f7800 x1874531139558656/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695994.750187:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695994.750189:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695994.750190:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695994.750193:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695994.750195:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695994.750197:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695994.750200:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7800 x1874531139558656/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695994.750210:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7800 x1874531139558656/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.750228:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695994.750230:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.750232:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695994.750233:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695994.750237:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7800 x1874531139558656/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.750245:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695994.750249:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7800 x1874531139558656/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.750257:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695994.750260:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.750261:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695994.750263:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.750265:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695994.750266:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695994.750268:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695994.750271:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695994.750274:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695994.750279:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7800 x1874531139558656/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695994.750287:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139558656:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787695994.750291:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.750293:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695994.750298:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6680 x1874531139558272/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.750305:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695994.750308:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695994.750313:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547800. 02000000:00000001:1.0:1787695994.750316:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695994.750319:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695994.750321:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695994.750324:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695994.750326:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:1.0:1787695994.750333:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695994.750335:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695994.750336:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6680. 00000100:00000001:1.0:1787695994.750340:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695994.750341:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.750343:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695994.750347:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4a80 x1874531139558400/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.750354:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695994.750356:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695994.750358:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547c00. 02000000:00000001:1.0:1787695994.750360:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695994.750361:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695994.750386:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787695994.750389:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695994.750391:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675800. 02000000:00000001:1.0:1787695994.750396:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695994.750399:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695994.750401:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4a80. 00000100:00000001:1.0:1787695994.750405:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695994.750407:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.750409:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695994.750414:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7b80 x1874531139558528/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.750421:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695994.750424:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695994.750426:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695994.750431:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695994.750433:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695994.750435:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787695994.750442:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695994.750444:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:1.0:1787695994.750449:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695994.750450:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695994.750452:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7b80. 00000100:00000001:1.0:1787695994.750455:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695994.750456:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.750458:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695994.750462:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7800 x1874531139558656/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.750468:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695994.750470:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695994.750472:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695994.750474:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695994.750475:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695994.750476:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787695994.750478:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695994.750480:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:1.0:1787695994.750482:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695994.750483:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695994.750484:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7800. 00000100:00000001:1.0:1787695994.750486:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695994.750488:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.750490:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.750496:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695994.750498:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695994.750500:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695994.750502:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787695994.750504:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695994.750509:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5c00 x1874531139558784/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.750530:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695994.750532:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695994.750534:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695994.750537:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695994.750540:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695994.750548:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139558784:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695994.750552:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695994.750555:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695994.750557:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695994.750559:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695994.750562:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695994.750564:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695994.750567:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695994.750570:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695994.750574:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695994.750577:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139558784, portal 10 00000100:00000001:1.0:1787695994.750579:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695994.750581:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150912 : -128555779400704 : ffff8b14466f5c00) 00000100:00000040:1.0:1787695994.750588:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5c00 x1874531139558784/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696019 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.750595:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695994.750598:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695994.750600:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695994.750602:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139558784, offset 0 00000400:00000200:1.0:1787695994.750607:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695994.750611:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695994.750614:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695994.750617:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695994.750620:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695994.750622:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695994.750625:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695994.750629:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17175:17175:256:4294967295] 192.168.201.140@tcp LPNI seq info [17174:17174:8:4294967295] 00000400:00000200:1.0:1787695994.750633:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695994.750641:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695994.750646:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695994.750650:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695994.750660:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695994.750663:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1787695994.750668:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695994.750670:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1787695994.750674:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695994.750677:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695994.750680:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695994.750684:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695994.750688:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695994.750690:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695994.750693:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695994.750697:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5c00 x1874531139558784/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696019 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.750705:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695994.750708:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5c00 x1874531139558784/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696019 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.750716:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695994.750718:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695994.750720:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695994.750724:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787695994.750728:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695994.750730:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298555101 00000100:00000001:1.0:1787695994.750733:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298566458 00000100:00000001:1.0:1787695994.750735:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298555101 00000100:00000001:1.0:1787695994.750738:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298566458 00000100:00000001:1.0:1787695994.750740:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695994.750742:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695994.750745:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695994.750749:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695994/real 1787695994] req@ffff8b14466f5c00 x1874531139558784/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695994.750758:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695994.750763:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695994.750765:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695994.750767:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695994.750769:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695994.750772:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695994.750776:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5c00 x1874531139558784/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695994.750785:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5c00 x1874531139558784/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.750792:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695994.750794:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.750796:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695994.750798:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695994.750802:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5c00 x1874531139558784/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.750810:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695994.750814:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5c00 x1874531139558784/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.750822:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695994.750824:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.750826:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695994.750828:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.750830:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695994.750832:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695994.750834:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695994.750838:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695994.750840:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695994.750845:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5c00 x1874531139558784/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695994.750865:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139558784:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695994.750869:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695994.750873:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6a00 x1874531139558912/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.750882:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787695994.750884:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695994.750886:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787695994.750888:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695994.750890:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695994.750893:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139558912:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695994.750897:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787695994.750899:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787695994.750901:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695994.750903:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787695994.750906:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695994.750908:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787695994.750912:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787695994.750914:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787695994.750918:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787695994.750921:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139558912, portal 10 00000100:00000001:1.0:1787695994.750923:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695994.750925:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930154496 : -128555779397120 : ffff8b14466f6a00) 00000100:00000040:1.0:1787695994.750931:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6a00 x1874531139558912/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696019 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.750938:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787695994.750940:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787695994.750943:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787695994.750945:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139558912, offset 0 00000400:00000200:1.0:1787695994.750949:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787695994.750957:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695994.750961:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787695994.750964:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787695994.750967:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787695994.750970:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695994.750972:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695994.750976:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17176:17176:256:4294967295] 192.168.201.140@tcp LPNI seq info [17175:17175:8:4294967295] 00000400:00000200:1.0:1787695994.750981:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695994.750988:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787695994.750992:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695994.750995:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787695994.750999:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787695994.751002:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 7 secs later) 00000800:00000200:1.1:1787695994.751007:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787695994.751009:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 7 secs later) 00000800:00000100:1.0:1787695994.751014:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787695994.751016:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787695994.751019:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787695994.751023:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787695994.751026:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787695994.751028:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787695994.751030:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787695994.751034:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6a00 x1874531139558912/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696019 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.751043:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695994.751046:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6a00 x1874531139558912/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696019 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.751054:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695994.751056:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695994.751059:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695994.751062:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787695994.751066:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787695994.751072:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298555079 00000100:00000001:1.0:1787695994.751074:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298566458 00000100:00000001:1.0:1787695994.751077:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298555079 00000100:00000001:1.0:1787695994.751079:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298566458 00000100:00000001:1.0:1787695994.751082:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787695994.751084:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695994.751086:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787695994.751091:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695994/real 1787695994] req@ffff8b14466f6a00 x1874531139558912/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787695994.751100:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787695994.751102:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787695994.751103:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787695994.751106:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787695994.751108:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787695994.751110:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787695994.751114:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6a00 x1874531139558912/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787695994.751122:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6a00 x1874531139558912/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.751129:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787695994.751131:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.751133:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695994.751135:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787695994.751138:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6a00 x1874531139558912/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.751146:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695994.751149:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6a00 x1874531139558912/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.751156:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695994.751164:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.751166:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695994.751167:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.751169:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787695994.751171:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787695994.751173:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787695994.751177:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787695994.751180:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787695994.751184:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6a00 x1874531139558912/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787695994.751193:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139558912:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787695994.751197:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.751199:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695994.751202:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5c00 x1874531139558784/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.751208:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695994.751210:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695994.751212:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544000. 02000000:00000001:1.0:1787695994.751215:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695994.751216:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695994.751218:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695994.751220:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695994.751221:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676000. 02000000:00000001:1.0:1787695994.751224:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695994.751225:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695994.751227:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5c00. 00000100:00000001:1.0:1787695994.751229:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695994.751230:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.751232:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787695994.751235:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6a00 x1874531139558912/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696019 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695994.751252:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695994.751254:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787695994.751255:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446544400. 02000000:00000001:1.0:1787695994.751258:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787695994.751259:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787695994.751260:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787695994.751262:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787695994.751264:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:1.0:1787695994.751266:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787695994.751268:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787695994.751269:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6a00. 00000100:00000001:1.0:1787695994.751272:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787695994.751273:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.751275:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695994.751279:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787695994.751280:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695994.751282:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787695994.751284:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695994.806472:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695994.806478:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695994.806479:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695994.806483:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695994.806486:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695994.806489:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695994.806491:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695994.806492:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695994.806494:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695995.446561:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695995.446571:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695995.446579:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695995.446580:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695995.446585:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695995.446586:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695995.446590:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695995.446591:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695995.446595:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695995.446596:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695995.446600:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695995.446601:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695995.446670:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695995.446675:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695995.446679:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695995.830478:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695995.830487:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695995.830489:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695995.830502:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695995.830507:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695995.830511:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695995.830513:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695995.830516:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695995.830518:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695996.470459:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695996.470467:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695996.470472:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695996.470473:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695996.470476:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695996.470480:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695996.470524:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695996.470531:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695996.470533:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695996.470534:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695996.470537:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695996.470539:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695996.470540:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695996.470542:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695996.470546:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695996.470547:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695996.470548:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695996.470549:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695996.470551:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695996.470551:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695996.470552:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695996.470553:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695996.470554:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695996.470555:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695996.470558:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695996.470559:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695996.470562:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695996.470564:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787695996.470565:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787695996.470567:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787695996.470568:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787695996.470569:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695996.470570:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787695996.470572:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695996.470573:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695996.470575:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695996.470645:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695996.470648:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695996.470651:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695996.854505:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695996.854513:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695996.854516:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695996.854523:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695996.854542:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695996.854547:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695996.854550:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695996.854552:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695996.854556:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695997.494497:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695997.494503:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695997.494507:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695997.494508:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695997.494510:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695997.494512:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695997.494514:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695997.494515:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695997.494518:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695997.494540:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695997.494544:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695997.494545:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695997.494626:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695997.494632:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695997.494635:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695997.878496:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695997.878504:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695997.878506:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695997.878511:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695997.878514:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695997.878518:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695997.878520:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695997.878523:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695997.878526:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695998.518506:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695998.518512:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695998.518517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695998.518518:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695998.518522:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695998.518523:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695998.518527:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695998.518528:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695998.518531:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695998.518532:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695998.518535:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695998.518537:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787695998.518588:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787695998.518591:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787695998.518593:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787695998.902475:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695998.902482:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695998.902483:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695998.902487:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695998.902490:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695998.902493:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695998.902500:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695998.902502:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695998.902503:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695999.542491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695999.542499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695999.542504:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695999.542506:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695999.542509:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695999.542511:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695999.542514:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695999.542515:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695999.542519:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695999.542520:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787695999.542523:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787695999.542524:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787695999.542580:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787695999.542583:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787695999.542585:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787695999.862502:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695999.862518:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787695999.862522:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695999.862527:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695999.862533:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695999.862543:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3887 00000400:00000200:1.0:1787695999.862556:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695999.862566:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695999.862571:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695999.862594:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695999.862599:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695999.862602:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695999.862606:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695999.862611:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695999.862614:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:1.0:1787695999.862630:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695999.862635:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:1.0:1787695999.862638:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:1.0:1787695999.862641:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:1.0:1787695999.862644:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695999.862647:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695999.862649:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695999.862651:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695999.862655:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:1.0:1787695999.862658:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695999.862660:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695999.862663:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695999.862667:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695999.862670:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695999.862673:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787695999.862676:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695999.862781:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665500. 00000020:00000040:1.0:1787695999.862787:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:1.0:1787695999.862792:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695999.862794:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695999.862798:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695999.862812:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695999.862818:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695999.862828:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446544400. 00000100:00000001:1.0:1787695999.862837:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695999.862847:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446665500 x1874531139559040/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695999.862865:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695999.862870:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695999.862876:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004ae1679a] to pc [ptlrpcd_rcv+-1] req@ffff8b1446665500 x1874531139559040/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695999.862902:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695999.862906:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695999.862909:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787695999.862910:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695999.862912:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695999.862915:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695999.862918:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3887 00000400:00000200:1.0:1787695999.862923:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695999.862927:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695999.862929:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695999.862933:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695999.862935:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695999.862938:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695999.862940:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:2.0:1787695999.862940:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1787695999.862946:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695999.862948:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000100:00000001:2.0:1787695999.862948:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1787695999.862950:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695999.862954:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000100:00000001:2.0:1787695999.862954:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1787695999.862959:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:1.0:1787695999.862963:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:2.0:1787695999.862965:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665500 x1874531139559040/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695999.862970:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695999.862974:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695999.862979:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695999.862985:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:2.0:1787695999.862985:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695999.862989:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695999.862994:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695999.862998:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695999.863002:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1787695999.863004:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00100000:2.0:1787695999.863007:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139559040:192.168.201.140@tcp:250:kworker.0 00000100:00080000:1.0:1787695999.863012:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695999.863015:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787695999.863017:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695999.863021:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787695999.863022:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695999.863026:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695999.863028:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695999.863029:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787695999.863034:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695999.863037:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000010:2.0:1787695999.863040:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445660c00. 00000008:00100000:1.0:1787695999.863043:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1787695999.863044:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1787695999.863050:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695999.863054:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695999.863056:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000010:1.0:1787695999.863067:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446667b80. 00000400:00000010:2.0:1787695999.863069:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000020:00000040:1.0:1787695999.863073:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:1.0:1787695999.863077:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695999.863079:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1787695999.863080:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000001:1.0:1787695999.863081:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695999.863085:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695999.863087:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1787695999.863092:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139559040, portal 25 02000000:00000010:1.0:1787695999.863093:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446544000. 00000100:00000001:2.0:1787695999.863097:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787695999.863100:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695999.863101:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929559296 : -128555779992320 : ffff8b1446665500) 00000100:00100000:1.0:1787695999.863109:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446667b80 x1874531139559168/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1787695999.863113:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665500 x1874531139559040/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695999.863131:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787695999.863134:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787695999.863135:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695999.863140:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000040:1.0:1787695999.863145:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006b2dcf49] to pc [ptlrpcd_rcv+-1] req@ffff8b1446667b80 x1874531139559168/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1787695999.863146:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695999.863151:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139559040, offset 0 00000400:00000200:2.0:1787695999.863161:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787695999.863171:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695999.863175:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:1.0:1787695999.863176:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695999.863181:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787695999.863184:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:2.0:1787695999.863185:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00080000:1.0:1787695999.863187:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695999.863192:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1787695999.863195:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695999.863203:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00080000:1.0:1787695999.863205:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3887 00000400:00000200:2.0:1787695999.863210:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787695999.863215:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695999.863217:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787695999.863223:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000200:2.0:1787695999.863225:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17177:17177:256:4294967295] 192.168.201.140@tcp LPNI seq info [17176:17176:8:4294967295] 00000100:00080000:1.0:1787695999.863228:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1787695999.863234:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:1.0:1787695999.863236:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695999.863240:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695999.863246:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695999.863251:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:2.0:1787695999.863252:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000020:00000001:1.0:1787695999.863256:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695999.863259:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:1.0:1787695999.863263:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000200:2.0:1787695999.863264:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000020:00000040:1.0:1787695999.863266:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:1.0:1787695999.863270:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000800:00000010:2.0:1787695999.863271:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000020:00000001:1.0:1787695999.863273:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:1.0:1787695999.863279:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695999.863284:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000200:2.0:1787695999.863285:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:1.0:1787695999.863289:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000200:2.0:1787695999.863291:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 2 secs later) 00000100:00000001:1.0:1787695999.863294:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695999.863300:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000800:00000200:2.1:1787695999.863304:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00080000:1.0:1787695999.863308:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000800:00000200:2.1:1787695999.863310:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 2 secs later) 00000100:00000001:1.0:1787695999.863313:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695999.863317:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:2.0:1787695999.863318:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 02000000:00000001:1.0:1787695999.863321:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:2.0:1787695999.863323:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 02000000:00000001:1.0:1787695999.863327:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695999.863331:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787695999.863333:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787695999.863336:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695999.863337:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695999.863341:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665f80. 00000020:00000040:1.0:1787695999.863344:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:1.0:1787695999.863346:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695999.863353:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695999.863355:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695999.863356:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695999.863358:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695999.863417:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446547c00. 00000100:00000001:1.0:1787695999.863436:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695999.863443:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446665f80 x1874531139559296/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695999.863451:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695999.863453:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695999.863456:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a0a2be0e] to pc [ptlrpcd_rcv+-1] req@ffff8b1446665f80 x1874531139559296/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695999.863464:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695999.863466:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695999.863469:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787695999.863471:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695999.863472:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695999.863475:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695999.863477:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3887 00000400:00000200:1.0:1787695999.863481:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695999.863485:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695999.863487:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695999.863489:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695999.863492:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695999.863494:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695999.863496:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695999.863498:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695999.863499:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:1.0:1787695999.863501:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695999.863503:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:1.0:1787695999.863505:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:1.0:1787695999.863516:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:1.0:1787695999.863520:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695999.863522:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695999.863524:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695999.863526:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695999.863529:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:1.0:1787695999.863531:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695999.863534:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695999.863536:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695999.863539:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695999.863541:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695999.863542:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787695999.863545:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787695999.863547:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695999.863548:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695999.863551:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446667100. 00000020:00000040:1.0:1787695999.863554:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:1.0:1787695999.863557:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695999.863558:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695999.863559:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695999.863560:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695999.863562:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695999.863565:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446547800. 00000100:00000001:1.0:1787695999.863568:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695999.863572:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446667100 x1874531139559424/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695999.863580:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695999.863582:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695999.863586:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f24be523] to pc [ptlrpcd_rcv+-1] req@ffff8b1446667100 x1874531139559424/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695999.863593:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695999.863601:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695999.863605:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1787695999.863606:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695999.863608:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695999.863611:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695999.863613:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3887 00000400:00000200:1.0:1787695999.863626:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695999.863630:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695999.863635:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695999.863638:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695999.863641:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695999.863643:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695999.863645:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787695999.863647:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695999.863648:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:1.0:1787695999.863650:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695999.863652:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:1.0:1787695999.863655:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:1.0:1787695999.863657:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:1.0:1787695999.863660:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695999.863662:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695999.863664:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695999.863665:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695999.863667:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:1.0:1787695999.863670:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695999.863672:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695999.863674:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695999.863677:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695999.863679:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695999.863681:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787695999.863701:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787695999.863703:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695999.863705:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695999.863708:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446664700. 00000020:00000040:1.0:1787695999.863711:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:1.0:1787695999.863713:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695999.863714:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695999.863716:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695999.863717:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695999.863719:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787695999.863723:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446546000. 00000100:00000001:1.0:1787695999.863726:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787695999.863730:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446664700 x1874531139559552/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787695999.863738:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787695999.863740:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695999.863745:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007106cb5] to pc [ptlrpcd_rcv+-1] req@ffff8b1446664700 x1874531139559552/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695999.863753:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787695999.863756:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695999.863759:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787695999.863761:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787695999.863762:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695999.863766:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787695999.863769:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3887 00000400:00000200:1.0:1787695999.863773:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787695999.863776:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787695999.863778:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787695999.863782:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695999.863784:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695999.863787:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:2.0:1787695999.863791:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:1.0:1787695999.863798:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:2.0:1787695999.863802:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000020:00000001:1.0:1787695999.863803:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695999.863807:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:1.0:1787695999.863809:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:2.0:1787695999.863812:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000020:00000040:1.0:1787695999.863813:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000400:00000010:2.0:1787695999.863817:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000020:00000001:1.0:1787695999.863818:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000020:00000001:1.0:1787695999.863822:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000040:1.0:1787695999.863829:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695999.863830:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787695999.863832:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695999.863840:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000200:2.0:1787695999.863840:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446665500 x1874531139559040/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695999.863845:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787695999.863852:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:1.0:1787695999.863857:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787695999.863859:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787695999.863861:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787695999.863864:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:2.0:1787695999.863866:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665500 x1874531139559040/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696024 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695999.863868:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695999.863873:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695999.863878:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787695999.863881:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787695999.863883:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1787695999.863886:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695999.863887:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787695999.863891:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787695999.863897:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665180. 00000100:00000001:2.0:1787695999.863897:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1787695999.863903:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000400:00000200:2.0:1787695999.863907:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:1.0:1787695999.863908:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787695999.863912:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695999.863912:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298571577 00000100:00000001:1.0:1787695999.863916:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787695999.863920:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787695999.863922:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695999.863924:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298566458 02000000:00000010:1.0:1787695999.863927:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446546800. 00000100:00000001:2.0:1787695999.863931:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298571577 00000100:00000001:1.0:1787695999.863933:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695999.863936:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298566458 00000100:00100000:1.0:1787695999.863942:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446665180 x1874531139559680/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787695999.863943:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695999.863950:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695999.863955:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695999.863961:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695999/real 1787695999] req@ffff8b1446665500 x1874531139559040/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695999.863981:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000001:2.0:1787695999.863985:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1787695999.863986:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787695999.863989:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695999.863994:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000100:00000040:1.0:1787695999.863995:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f923372e] to pc [ptlrpcd_rcv+-1] req@ffff8b1446665180 x1874531139559680/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787695999.863998:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695999.864002:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695999.864007:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787695999.864013:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1787695999.864014:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665500 x1874531139559040/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695999.864018:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787695999.864028:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787695999.864032:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000200:2.0:1787695999.864033:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665500 x1874531139559040/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787695999.864035:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787695999.864043:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787695999.864047:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:1.0:1787695999.864049:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3887 00000400:00000200:1.0:1787695999.864055:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695999.864056:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1787695999.864061:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:2.0:1787695999.864062:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:1.0:1787695999.864066:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787695999.864067:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695999.864075:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00100000:2.0:1787695999.864077:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446665500 x1874531139559040/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787695999.864080:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787695999.864086:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787695999.864091:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:2.0:1787695999.864093:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787695999.864096:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787695999.864099:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000100:00000040:2.0:1787695999.864101:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665500 x1874531139559040/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787695999.864102:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787695999.864106:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:1.0:1787695999.864111:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000020:00000001:1.0:1787695999.864117:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:2.0:1787695999.864118:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695999.864124:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695999.864125:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695999.864127:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787695999.864130:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787695999.864131:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787695999.864135:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787695999.864136:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695999.864140:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787695999.864141:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00080000:2.0:1787695999.864145:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000020:00000040:1.0:1787695999.864152:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00080000:2.0:1787695999.864154:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695999.864159:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:1.0:1787695999.864160:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787695999.864165:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695999.864167:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665500 x1874531139559040/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787695999.864168:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787695999.864172:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787695999.864177:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787695999.864182:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:2.0:1787695999.864185:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139559040:192.168.201.140@tcp:250:kworker.0 00000008:00100000:1.0:1787695999.864187:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787695999.864193:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695999.864194:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695999.864196:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695999.864198:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000010:1.0:1787695999.864202:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446667800. 00000100:00000040:2.0:1787695999.864204:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665500 x1874531139559040/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1787695999.864208:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:1.0:1787695999.864213:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787695999.864216:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787695999.864217:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695999.864220:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1787695999.864221:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000010:2.0:1787695999.864224:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445660c00. 02000000:00000001:1.0:1787695999.864225:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787695999.864229:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:1.0:1787695999.864230:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787695999.864234:0:2402:0:(genops.c:1217:class_import_put()) Process entered 02000000:00000010:1.0:1787695999.864237:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545400. 00000100:00000001:1.0:1787695999.864241:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1787695999.864242:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:2.0:1787695999.864248:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00100000:1.0:1787695999.864252:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446667800 x1874531139559808/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:2.0:1787695999.864252:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446544400. 02000000:00000001:2.0:1787695999.864261:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1787695999.864263:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1787695999.864265:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787695999.864268:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787695999.864269:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665500. 00000100:00000001:2.0:1787695999.864275:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000040:1.0:1787695999.864276:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009df211f4] to pc [ptlrpcd_rcv+-1] req@ffff8b1446667800 x1874531139559808/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.864278:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.864284:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787695999.864292:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695999.864294:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787695999.864295:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695999.864299:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787695999.864300:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:2.0:1787695999.864305:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1787695999.864306:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787695999.864308:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787695999.864311:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787695999.864314:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1787695999.864318:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446667b80 x1874531139559168/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787695999.864329:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3897) 00000100:00000001:2.0:1787695999.865472:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695999.865474:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695999.865478:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695999.865479:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695999.865482:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695999.865486:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446667b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139559168:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695999.865490:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695999.865492:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695999.865494:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695999.865496:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695999.865500:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445660c00. 02000000:00000001:2.0:1787695999.865502:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695999.865507:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695999.865511:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695999.865516:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695999.865521:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139559168, portal 10 00000100:00000001:2.0:1787695999.865523:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695999.865524:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929569152 : -128555779982464 : ffff8b1446667b80) 00000100:00000040:2.0:1787695999.865531:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446667b80 x1874531139559168/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.865545:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695999.865548:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695999.865551:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695999.865554:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139559168, offset 0 00000400:00000200:2.0:1787695999.865560:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695999.865565:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695999.865570:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695999.865574:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695999.865577:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695999.865579:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695999.865582:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695999.865585:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17178:17178:256:4294967295] 192.168.201.140@tcp LPNI seq info [17177:17177:8:4294967295] 00000400:00000200:2.0:1787695999.865589:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695999.865596:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695999.865601:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695999.865604:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695999.865610:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695999.865613:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1787695999.865617:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695999.865620:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1787695999.865624:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695999.865626:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695999.865629:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695999.865632:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695999.865636:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695999.865638:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695999.865641:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695999.865644:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446667b80 x1874531139559168/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.865652:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695999.865655:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446667b80 x1874531139559168/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696024 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.865672:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695999.865674:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695999.865678:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695999.865682:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000400:00000200:2.0:1787695999.865686:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695999.865688:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298561480 00000100:00000001:2.0:1787695999.865690:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298571577 00000100:00000001:2.0:1787695999.865692:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298561480 00000100:00000001:2.0:1787695999.865695:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298571577 00000100:00000001:2.0:1787695999.865697:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695999.865699:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695999.865701:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695999.865705:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695999/real 1787695999] req@ffff8b1446667b80 x1874531139559168/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695999.865713:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695999.865715:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695999.865716:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695999.865718:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695999.865720:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695999.865723:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695999.865726:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446667b80 x1874531139559168/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695999.865733:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446667b80 x1874531139559168/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.865739:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695999.865741:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.865743:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695999.865745:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695999.865748:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446667b80 x1874531139559168/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.865759:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695999.865764:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446667b80 x1874531139559168/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.865770:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695999.865773:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.865774:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695999.865776:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.865778:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695999.865780:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695999.865783:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695999.865786:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695999.865788:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695999.865792:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446667b80 x1874531139559168/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695999.865799:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446667b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139559168:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695999.865803:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695999.865807:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665f80 x1874531139559296/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.865812:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695999.865814:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695999.865816:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695999.865818:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695999.865820:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695999.865822:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139559296:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695999.865826:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695999.865827:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695999.865833:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695999.865835:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695999.865838:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445661400. 02000000:00000001:2.0:1787695999.865840:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695999.865843:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695999.865846:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695999.865849:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695999.865852:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139559296, portal 10 00000100:00000001:2.0:1787695999.865854:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695999.865855:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929561984 : -128555779989632 : ffff8b1446665f80) 00000100:00000040:2.0:1787695999.865861:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665f80 x1874531139559296/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.865867:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695999.865869:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695999.865872:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695999.865874:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139559296, offset 0 00000400:00000200:2.0:1787695999.865878:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695999.865882:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695999.865885:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695999.865888:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695999.865890:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695999.865893:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695999.865895:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695999.865898:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17179:17179:256:4294967295] 192.168.201.140@tcp LPNI seq info [17178:17178:8:4294967295] 00000400:00000200:2.0:1787695999.865901:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695999.865907:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695999.865911:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695999.865914:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695999.865918:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695999.865921:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1787695999.865925:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695999.865927:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1787695999.865931:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695999.865937:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695999.865940:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695999.865943:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695999.865946:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695999.865948:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695999.865950:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695999.865954:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446665f80 x1874531139559296/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.865962:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695999.865964:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665f80 x1874531139559296/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696024 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.865970:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695999.865972:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695999.865975:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695999.865978:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:2.0:1787695999.865981:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695999.865982:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298560328 00000100:00000001:2.0:1787695999.865985:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298571577 00000100:00000001:2.0:1787695999.865987:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298560328 00000100:00000001:2.0:1787695999.865989:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298571577 00000100:00000001:2.0:1787695999.865991:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695999.865994:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695999.865995:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695999.865999:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695999/real 1787695999] req@ffff8b1446665f80 x1874531139559296/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695999.866006:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695999.866008:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695999.866009:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695999.866011:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695999.866013:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695999.866020:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695999.866023:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665f80 x1874531139559296/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695999.866031:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665f80 x1874531139559296/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.866036:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695999.866038:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.866040:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695999.866041:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695999.866045:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446665f80 x1874531139559296/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.866051:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695999.866055:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665f80 x1874531139559296/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.866061:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695999.866063:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.866065:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695999.866066:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.866068:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695999.866069:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695999.866071:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695999.866074:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695999.866077:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695999.866080:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665f80 x1874531139559296/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695999.866087:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139559296:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695999.866091:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695999.866098:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446667100 x1874531139559424/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.866105:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695999.866106:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695999.866108:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695999.866110:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695999.866112:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695999.866114:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446667100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139559424:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787695999.866117:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695999.866119:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695999.866121:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695999.866123:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695999.866126:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445663000. 02000000:00000001:2.0:1787695999.866128:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695999.866131:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695999.866134:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695999.866137:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695999.866140:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139559424, portal 4 00000100:00000001:2.0:1787695999.866142:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695999.866143:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929566464 : -128555779985152 : ffff8b1446667100) 00000100:00000040:2.0:1787695999.866161:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446667100 x1874531139559424/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.866171:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695999.866173:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695999.866175:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695999.866177:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139559424, offset 0 00000400:00000200:2.0:1787695999.866181:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695999.866184:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695999.866187:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695999.866190:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695999.866193:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695999.866195:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695999.866197:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695999.866206:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17180:17180:256:4294967295] 192.168.201.140@tcp LPNI seq info [17179:17179:8:4294967295] 00000400:00000200:2.0:1787695999.866210:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695999.866216:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695999.866221:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695999.866224:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695999.866228:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695999.866231:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1787695999.866235:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695999.866241:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1787695999.866244:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695999.866246:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695999.866249:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695999.866253:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695999.866256:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695999.866258:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695999.866260:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695999.866264:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446667100 x1874531139559424/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.866271:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695999.866273:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446667100 x1874531139559424/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696024 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.866279:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695999.866281:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695999.866284:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695999.866287:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:2.0:1787695999.866290:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695999.866292:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298560358 00000100:00000001:2.0:1787695999.866295:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298571577 00000100:00000001:2.0:1787695999.866297:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298560358 00000100:00000001:2.0:1787695999.866299:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298571577 00000100:00000001:2.0:1787695999.866301:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695999.866308:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695999.866309:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695999.866313:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695999/real 1787695999] req@ffff8b1446667100 x1874531139559424/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695999.866321:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695999.866322:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695999.866324:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695999.866326:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695999.866328:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695999.866468:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695999.866473:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446667100 x1874531139559424/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695999.866481:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446667100 x1874531139559424/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.866487:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695999.866489:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.866492:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695999.866493:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695999.866497:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446667100 x1874531139559424/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.866505:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695999.866510:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446667100 x1874531139559424/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.866528:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695999.866531:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.866532:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695999.866534:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.866535:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695999.866537:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695999.866539:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695999.866548:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695999.866550:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695999.866554:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446667100 x1874531139559424/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695999.866564:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446667100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139559424:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787695999.866568:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695999.866572:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446664700 x1874531139559552/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.866578:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695999.866580:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695999.866582:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695999.866584:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695999.866586:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695999.866588:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446664700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139559552:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787695999.866591:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695999.866593:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695999.866595:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695999.866596:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695999.866600:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445661000. 02000000:00000001:2.0:1787695999.866602:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695999.866605:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695999.866608:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695999.866611:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695999.866615:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139559552, portal 4 00000100:00000001:2.0:1787695999.866616:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695999.866618:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929555712 : -128555779995904 : ffff8b1446664700) 00000100:00000040:2.0:1787695999.866623:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446664700 x1874531139559552/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.866629:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695999.866636:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695999.866638:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695999.866640:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139559552, offset 0 00000400:00000200:2.0:1787695999.866644:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695999.866648:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695999.866652:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695999.866654:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695999.866656:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695999.866659:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695999.866661:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695999.866664:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17181:17181:256:4294967295] 192.168.201.140@tcp LPNI seq info [17180:17180:8:4294967295] 00000400:00000200:2.0:1787695999.866667:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695999.866673:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695999.866677:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695999.866680:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695999.866684:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695999.866687:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1787695999.866690:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695999.866693:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1787695999.866697:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695999.866699:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695999.866702:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695999.866705:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695999.866708:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695999.866709:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695999.866712:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695999.866715:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446664700 x1874531139559552/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.866722:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695999.866725:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446664700 x1874531139559552/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696024 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.866731:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695999.866732:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695999.866741:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695999.866745:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:2.0:1787695999.866749:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695999.866751:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298560031 00000100:00000001:2.0:1787695999.866753:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298571577 00000100:00000001:2.0:1787695999.866755:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298560031 00000100:00000001:2.0:1787695999.866757:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298571577 00000100:00000001:2.0:1787695999.866759:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695999.866762:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695999.866764:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695999.866767:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695999/real 1787695999] req@ffff8b1446664700 x1874531139559552/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695999.866775:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695999.866776:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695999.866778:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695999.866781:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695999.866783:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695999.866784:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695999.866787:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446664700 x1874531139559552/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695999.866795:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446664700 x1874531139559552/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.866801:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695999.866802:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.866804:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695999.866806:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695999.866809:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446664700 x1874531139559552/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.866815:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695999.866819:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446664700 x1874531139559552/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.866830:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695999.866832:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.866834:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695999.866835:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.866837:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695999.866839:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695999.866841:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695999.866844:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695999.866846:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695999.866850:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446664700 x1874531139559552/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695999.866857:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446664700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139559552:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787695999.866861:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695999.866865:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665180 x1874531139559680/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.866870:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695999.866872:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695999.866874:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695999.866875:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695999.866877:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695999.866880:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139559680:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695999.866883:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695999.866884:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695999.866886:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695999.866888:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695999.866891:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445663400. 02000000:00000001:2.0:1787695999.866893:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695999.866896:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695999.866903:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695999.866907:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695999.866910:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139559680, portal 10 00000100:00000001:2.0:1787695999.866912:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695999.866913:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929558400 : -128555779993216 : ffff8b1446665180) 00000100:00000040:2.0:1787695999.866919:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665180 x1874531139559680/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.866925:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695999.866927:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695999.866929:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695999.866931:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139559680, offset 0 00000400:00000200:2.0:1787695999.866935:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695999.866938:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695999.866941:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695999.866944:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695999.866946:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695999.866949:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695999.866951:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695999.866954:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17182:17182:256:4294967295] 192.168.201.140@tcp LPNI seq info [17181:17181:8:4294967295] 00000400:00000200:2.0:1787695999.866958:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695999.866963:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695999.866968:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695999.866971:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695999.866975:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695999.866977:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1787695999.866981:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695999.866984:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1787695999.866987:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695999.866989:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695999.866992:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695999.866995:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695999.866998:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695999.866999:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695999.867006:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695999.867010:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446665180 x1874531139559680/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.867017:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695999.867019:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665180 x1874531139559680/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696024 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.867025:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695999.867027:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695999.867030:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695999.867033:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:2.0:1787695999.867036:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695999.867038:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298560217 00000100:00000001:2.0:1787695999.867041:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298571577 00000100:00000001:2.0:1787695999.867043:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298560217 00000100:00000001:2.0:1787695999.867045:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298571577 00000100:00000001:2.0:1787695999.867047:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695999.867049:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695999.867051:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695999.867054:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695999/real 1787695999] req@ffff8b1446665180 x1874531139559680/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695999.867061:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695999.867062:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695999.867064:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695999.867066:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695999.867068:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695999.867070:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695999.867073:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665180 x1874531139559680/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695999.867080:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665180 x1874531139559680/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.867095:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695999.867097:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.867099:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695999.867100:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695999.867104:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446665180 x1874531139559680/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.867110:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695999.867113:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665180 x1874531139559680/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.867120:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695999.867122:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.867123:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695999.867125:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.867126:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695999.867128:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695999.867130:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695999.867133:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695999.867135:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695999.867139:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665180 x1874531139559680/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695999.867146:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139559680:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695999.867150:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787695999.867154:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446667800 x1874531139559808/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.867161:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787695999.867162:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695999.867168:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787695999.867170:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787695999.867172:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787695999.867175:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446667800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139559808:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695999.867178:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787695999.867180:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787695999.867182:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787695999.867184:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787695999.867186:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445660400. 02000000:00000001:2.0:1787695999.867188:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787695999.867192:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787695999.867194:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787695999.867197:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787695999.867200:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139559808, portal 10 00000100:00000001:2.0:1787695999.867202:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787695999.867203:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929568256 : -128555779983360 : ffff8b1446667800) 00000100:00000040:2.0:1787695999.867209:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446667800 x1874531139559808/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.867214:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787695999.867216:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787695999.867218:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000200:2.0:1787695999.867220:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139559808, offset 0 00000400:00000200:2.0:1787695999.867224:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787695999.867227:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695999.867230:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787695999.867233:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787695999.867235:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787695999.867238:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787695999.867240:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787695999.867243:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17183:17183:256:4294967295] 192.168.201.140@tcp LPNI seq info [17182:17182:8:4294967295] 00000400:00000200:2.0:1787695999.867246:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787695999.867252:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787695999.867261:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695999.867264:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000800:00000200:2.0:1787695999.867267:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787695999.867270:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 2 secs later) 00000800:00000200:2.1:1787695999.867273:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787695999.867276:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 2 secs later) 00000800:00000100:2.0:1787695999.867280:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787695999.867282:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:2.0:1787695999.867284:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787695999.867288:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787695999.867290:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:2.0:1787695999.867292:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:2.0:1787695999.867295:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787695999.867298:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446667800 x1874531139559808/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696024 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.867305:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695999.867307:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446667800 x1874531139559808/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696024 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.867313:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695999.867315:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787695999.867318:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787695999.867320:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:2.0:1787695999.867324:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787695999.867326:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298560195 00000100:00000001:2.0:1787695999.867334:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298571577 00000100:00000001:2.0:1787695999.867337:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298560195 00000100:00000001:2.0:1787695999.867339:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298571577 00000100:00000001:2.0:1787695999.867341:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787695999.867343:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695999.867345:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787695999.867349:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787695999/real 1787695999] req@ffff8b1446667800 x1874531139559808/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787695999.867361:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787695999.867377:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787695999.867379:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787695999.867382:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787695999.867384:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787695999.867386:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787695999.867391:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446667800 x1874531139559808/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787695999.867400:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446667800 x1874531139559808/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.867407:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787695999.867408:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.867410:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695999.867412:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787695999.867415:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446667800 x1874531139559808/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.867422:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787695999.867436:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446667800 x1874531139559808/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.867443:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787695999.867445:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.867447:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787695999.867448:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.867450:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787695999.867451:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787695999.867454:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787695999.867457:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787695999.867459:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787695999.867468:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446667800 x1874531139559808/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787695999.867475:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446667800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139559808:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787695999.867479:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.867481:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695999.867485:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446667b80 x1874531139559168/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.867491:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695999.867492:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695999.867494:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445660c00. 02000000:00000001:2.0:1787695999.867497:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695999.867499:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695999.867501:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:2.0:1787695999.867503:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695999.867505:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446544000. 02000000:00000001:2.0:1787695999.867509:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695999.867511:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695999.867512:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446667b80. 00000100:00000001:2.0:1787695999.867515:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695999.867517:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.867518:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695999.867522:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665f80 x1874531139559296/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.867528:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695999.867529:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695999.867530:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445661400. 02000000:00000001:2.0:1787695999.867533:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695999.867534:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695999.867535:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:2.0:1787695999.867538:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695999.867539:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446547c00. 02000000:00000001:2.0:1787695999.867542:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695999.867549:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695999.867551:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665f80. 00000100:00000001:2.0:1787695999.867554:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695999.867555:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.867557:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695999.867560:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446667100 x1874531139559424/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.867566:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695999.867567:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695999.867569:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445663000. 02000000:00000001:2.0:1787695999.867571:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695999.867573:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695999.867574:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:2.0:1787695999.867576:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695999.867577:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446547800. 02000000:00000001:2.0:1787695999.867579:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695999.867581:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695999.867582:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446667100. 00000100:00000001:2.0:1787695999.867584:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695999.867585:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.867587:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695999.867590:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446664700 x1874531139559552/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.867596:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695999.867597:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695999.867599:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445661000. 02000000:00000001:2.0:1787695999.867601:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695999.867602:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695999.867603:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:2.0:1787695999.867605:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695999.867606:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446546000. 02000000:00000001:2.0:1787695999.867610:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695999.867611:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695999.867613:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446664700. 00000100:00000001:2.0:1787695999.867615:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695999.867616:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.867622:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695999.867625:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665180 x1874531139559680/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.867631:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695999.867633:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695999.867635:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445663400. 02000000:00000001:2.0:1787695999.867637:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695999.867638:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695999.867639:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:2.0:1787695999.867641:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695999.867642:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446546800. 02000000:00000001:2.0:1787695999.867645:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695999.867646:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695999.867647:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665180. 00000100:00000001:2.0:1787695999.867649:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695999.867651:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.867652:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787695999.867655:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446667800 x1874531139559808/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696024 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787695999.867661:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787695999.867662:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787695999.867663:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445660400. 02000000:00000001:2.0:1787695999.867665:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787695999.867666:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787695999.867667:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:2.0:1787695999.867669:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787695999.867670:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545400. 02000000:00000001:2.0:1787695999.867674:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787695999.867675:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787695999.867676:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446667800. 00000100:00000001:2.0:1787695999.867678:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787695999.867680:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.867681:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787695999.867687:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787695999.867688:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787695999.867691:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787695999.867696:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787695999.926452:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787695999.926458:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787695999.926459:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787695999.926463:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787695999.926465:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787695999.926469:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695999.926470:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787695999.926472:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787695999.926473:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696000.566492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696000.566499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696000.566504:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696000.566505:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696000.566508:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696000.566509:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696000.566512:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696000.566512:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696000.566515:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696000.566515:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696000.566517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696000.566518:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787696000.566580:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787696000.566584:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787696000.566588:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787696000.950459:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787696000.950465:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787696000.950467:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787696000.950484:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787696000.950486:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787696000.950489:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787696000.950491:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787696000.950493:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787696000.950495:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696001.590440:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696001.590445:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696001.590449:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696001.590450:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696001.590452:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696001.590453:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696001.590455:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696001.590456:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696001.590458:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696001.590460:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787696001.590462:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787696001.590466:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787696001.590468:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787696001.590469:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696001.590472:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787696001.590473:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696001.590475:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696001.590476:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696001.590478:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696001.590479:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787696001.590521:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787696001.590525:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787696001.590528:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787696001.974460:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787696001.974467:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787696001.974469:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787696001.974475:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787696001.974479:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787696001.974482:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787696001.974485:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787696001.974487:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787696001.974498:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696002.614470:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696002.614476:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696002.614482:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696002.614483:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696002.614487:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696002.614505:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696002.614509:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696002.614510:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696002.614513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696002.614514:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696002.614517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696002.614519:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787696002.614578:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787696002.614581:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787696002.614584:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787696002.998479:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787696002.998486:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787696002.998488:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787696002.998493:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787696002.998497:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787696002.998501:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787696002.998504:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787696002.998506:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787696002.998509:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696003.638530:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696003.638540:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696003.638547:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696003.638549:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696003.638553:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696003.638554:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696003.638558:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696003.638560:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696003.638563:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696003.638565:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696003.638568:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696003.638569:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787696003.638657:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787696003.638664:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787696003.638667:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1787696004.022419:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787696004.022425:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787696004.022427:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787696004.022431:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787696004.022433:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787696004.022436:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787696004.022438:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787696004.022439:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787696004.022441:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696004.662469:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696004.662477:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787696004.662479:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787696004.662482:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696004.662487:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787696004.662489:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696004.662491:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696004.662493:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696004.662498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696004.662499:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787696004.662501:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787696004.662502:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696004.662504:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787696004.662505:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696004.662506:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696004.662508:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696004.662512:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696004.662513:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696004.662517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696004.662533:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696004.662537:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696004.662538:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696004.662541:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696004.662542:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787696004.662594:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787696004.662598:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787696004.662606:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787696004.982458:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787696004.982470:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787696004.982473:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787696004.982477:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787696004.982482:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787696004.982490:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3892 00000400:00000200:1.0:1787696004.982499:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696004.982505:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787696004.982508:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787696004.982511:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696004.982519:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787696004.982521:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696004.982523:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787696004.982527:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787696004.982529:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:1.0:1787696004.982532:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787696004.982535:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:1.0:1787696004.982538:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:1.0:1787696004.982540:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:1.0:1787696004.982543:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696004.982544:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787696004.982546:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696004.982548:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787696004.982550:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:1.0:1787696004.982553:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787696004.982555:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696004.982565:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787696004.982569:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696004.982573:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696004.982575:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787696004.982577:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787696004.982583:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665880. 00000020:00000040:1.0:1787696004.982587:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:1.0:1787696004.982591:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787696004.982593:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696004.982596:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787696004.982598:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787696004.982603:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787696004.982610:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545800. 00000100:00000001:1.0:1787696004.982617:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696004.982627:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446665880 x1874531139559936/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787696004.982635:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787696004.982640:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696004.982645:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000716595a9] to pc [ptlrpcd_rcv+-1] req@ffff8b1446665880 x1874531139559936/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696004.982663:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787696004.982666:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787696004.982669:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787696004.982670:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787696004.982672:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787696004.982675:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787696004.982678:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3892 00000400:00000200:1.0:1787696004.982682:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696004.982686:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787696004.982689:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787696004.982689:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1787696004.982695:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696004.982696:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787696004.982698:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787696004.982700:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787696004.982708:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665880 x1874531139559936/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787696004.982710:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696004.982715:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787696004.982720:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787696004.982722:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:1.0:1787696004.982725:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787696004.982727:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000040:1.0:1787696004.982729:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000100:00000001:2.0:1787696004.982731:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787696004.982735:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 02000000:00000001:2.0:1787696004.982737:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1787696004.982738:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 02000000:00000001:2.0:1787696004.982741:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1787696004.982742:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696004.982746:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1787696004.982746:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696004.982751:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00100000:2.0:1787696004.982752:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139559936:192.168.201.140@tcp:250:kworker.0 00000100:00000001:1.0:1787696004.982754:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787696004.982759:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00000001:2.0:1787696004.982760:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:1.0:1787696004.982764:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000001:2.0:1787696004.982766:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1787696004.982767:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696004.982770:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787696004.982770:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696004.982774:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787696004.982775:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787696004.982779:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696004.982780:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000010:2.0:1787696004.982783:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445660400. 00000008:00100000:1.0:1787696004.982784:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787696004.982788:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696004.982788:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696004.982793:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787696004.982796:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446664a80. 00000400:00000200:2.0:1787696004.982798:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000040:1.0:1787696004.982799:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000400:00000010:2.0:1787696004.982805:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 02000000:00000001:1.0:1787696004.982812:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787696004.982815:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1787696004.982815:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000001:1.0:1787696004.982818:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787696004.982820:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787696004.982822:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1787696004.982825:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139559936, portal 25 02000000:00000010:1.0:1787696004.982826:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545000. 00000100:00000001:1.0:1787696004.982832:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696004.982832:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787696004.982835:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929560192 : -128555779991424 : ffff8b1446665880) 00000100:00100000:1.0:1787696004.982838:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446664a80 x1874531139560064/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1787696004.982844:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665880 x1874531139559936/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696004.982855:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787696004.982858:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696004.982863:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787696004.982865:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ad35ba92] to pc [ptlrpcd_rcv+-1] req@ffff8b1446664a80 x1874531139560064/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787696004.982871:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787696004.982876:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8e58. 00000100:00000001:1.0:1787696004.982877:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1787696004.982880:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139559936, offset 0 00000100:00080000:1.0:1787696004.982881:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787696004.982885:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787696004.982886:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787696004.982889:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:2.0:1787696004.982889:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787696004.982894:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787696004.982897:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3892 00000400:00000200:2.0:1787696004.982900:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696004.982904:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696004.982908:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787696004.982911:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000200:2.0:1787696004.982915:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00080000:1.0:1787696004.982916:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787696004.982920:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696004.982925:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787696004.982929:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:2.0:1787696004.982929:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:1.0:1787696004.982934:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:2.0:1787696004.982937:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000020:00000001:1.0:1787696004.982938:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787696004.982941:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000400:00000200:2.0:1787696004.982944:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000020:00000001:1.0:1787696004.982945:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787696004.982948:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000400:00000200:2.0:1787696004.982952:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17184:17184:256:4294967295] 192.168.201.140@tcp LPNI seq info [17183:17183:8:4294967295] 00000020:00000001:1.0:1787696004.982953:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:1.0:1787696004.982958:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:1.0:1787696004.982964:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:2.0:1787696004.982964:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696004.982975:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1787696004.982975:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000040:1.0:1787696004.982982:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000200:2.0:1787696004.982985:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787696004.982986:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000010:2.0:1787696004.982990:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5400. 00000020:00000040:1.0:1787696004.982995:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000800:00000200:2.0:1787696004.982998:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00080000:1.0:1787696004.983001:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787696004.983007:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1787696004.983007:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:1.0:1787696004.983012:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787696004.983015:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696004.983017:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696004.983018:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787696004.983020:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787696004.983021:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696004.983023:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787696004.983025:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446664000. 00000100:00000001:2.0:1787696004.983026:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1787696004.983027:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:1.0:1787696004.983032:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1787696004.983034:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:1.0:1787696004.983035:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696004.983038:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787696004.983038:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298576696 02000000:00000001:1.0:1787696004.983040:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787696004.983043:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696004.983046:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298571577 02000000:00000010:1.0:1787696004.983047:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446544c00. 00000100:00000001:1.0:1787696004.983051:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696004.983054:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298576696 00000100:00100000:1.0:1787696004.983057:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446664000 x1874531139560192/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787696004.983060:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298571577 00000100:00000001:1.0:1787696004.983067:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787696004.983067:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696004.983070:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696004.983073:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696004.983074:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e60c2a2d] to pc [ptlrpcd_rcv+-1] req@ffff8b1446664000 x1874531139560192/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696004.983078:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696004.983082:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787696004.983084:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787696004.983084:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787696004.983089:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787696004.983090:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787696004.983092:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787696004.983094:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787696004.983094:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787696004.983097:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00080000:1.0:1787696004.983102:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3892 00000100:00000001:2.0:1787696004.983102:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:1.0:1787696004.983106:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787696004.983106:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787696004.983108:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000400:00000200:1.0:1787696004.983111:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787696004.983113:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787696004.983117:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446664a80 x1874531139560064/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787696004.983118:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696004.983121:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787696004.983123:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696004.983124:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787696004.983125:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787696004.983126:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:1.0:1787696004.983127:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787696004.983129:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:1.0:1787696004.983130:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:1.0:1787696004.983133:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:1.0:1787696004.983135:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696004.983138:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787696004.983142:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696004.983145:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787696004.983150:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00000001:2.0:1787696004.983150:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787696004.983154:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787696004.983155:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000001:2.0:1787696004.983158:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1787696004.983159:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696004.983162:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787696004.983163:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696004.983164:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787696004.983168:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696004.983169:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696004.983172:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787696004.983175:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787696004.983175:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446664a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139560064:192.168.201.140@tcp:38:kworker.0 00000008:00000001:1.0:1787696004.983178:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696004.983180:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787696004.983187:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446666680. 00000100:00000001:2.0:1787696004.983187:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787696004.983189:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:1.0:1787696004.983192:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:2.0:1787696004.983193:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696004.983197:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787696004.983197:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787696004.983201:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445663400. 02000000:00000001:2.0:1787696004.983203:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696004.983208:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696004.983209:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696004.983212:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000010:2.0:1787696004.983215:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752b688. 02000000:00000001:1.0:1787696004.983216:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787696004.983220:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1787696004.983222:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8bb0. 02000000:00000010:1.0:1787696004.983226:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446547400. 00000100:00000200:2.0:1787696004.983227:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139560064, portal 10 00000100:00000001:2.0:1787696004.983231:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787696004.983232:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696004.983233:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929556608 : -128555779995008 : ffff8b1446664a80) 00000100:00100000:1.0:1787696004.983243:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446666680 x1874531139560320/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1787696004.983245:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446664a80 x1874531139560064/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696004.983268:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787696004.983271:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000040:2.0:1787696004.983272:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787696004.983276:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8440. 00000100:00000001:1.0:1787696004.983277:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1787696004.983279:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139560064, offset 0 00000100:00000040:1.0:1787696004.983285:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004bd4d567] to pc [ptlrpcd_rcv+-1] req@ffff8b1446666680 x1874531139560320/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787696004.983285:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787696004.983291:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696004.983298:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787696004.983303:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1787696004.983304:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696004.983308:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00080000:1.0:1787696004.983309:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1787696004.983313:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00080000:1.0:1787696004.983316:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000200:2.0:1787696004.983318:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:1.0:1787696004.983320:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:2.0:1787696004.983324:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17185:17185:255:4294967295] 192.168.201.140@tcp LPNI seq info [17184:17184:7:4294967295] 00000100:00080000:1.0:1787696004.983325:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787696004.983330:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1787696004.983334:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:1.0:1787696004.983335:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3892 00000400:00000200:1.0:1787696004.983356:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696004.983387:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787696004.983393:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787696004.983395:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787696004.983399:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787696004.983403:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696004.983405:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787696004.983407:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000800:00000010:2.0:1787696004.983409:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5300. 00000100:00000001:1.0:1787696004.983410:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787696004.983415:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787696004.983418:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000800:00000200:2.0:1787696004.983419:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000020:00000001:1.0:1787696004.983421:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787696004.983423:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:1.0:1787696004.983429:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000800:00000200:2.1:1787696004.983433:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:2.0:1787696004.983437:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787696004.983439:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:2.0:1787696004.983442:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000100:00000040:1.0:1787696004.983445:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:2.0:1787696004.983450:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696004.983451:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787696004.983455:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696004.983455:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298566598 00000100:00000001:1.0:1787696004.983461:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:2.0:1787696004.983461:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298576696 00000100:00000001:2.0:1787696004.983466:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298566598 00000020:00000040:1.0:1787696004.983467:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00000001:2.0:1787696004.983472:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298576696 00000100:00080000:1.0:1787696004.983473:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787696004.983478:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696004.983479:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696004.983483:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696004.983484:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787696004.983489:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787696004.983489:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1787696004.983493:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787696004.983496:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446664000 x1874531139560192/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696004.983497:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787696004.983500:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787696004.983506:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696004.983511:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696004.983513:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787696004.983515:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787696004.983517:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446667800. 02000000:00000001:2.0:1787696004.983518:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:1.0:1787696004.983523:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:2.0:1787696004.983523:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787696004.983528:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696004.983529:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787696004.983533:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787696004.983534:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446664000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139560192:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696004.983538:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787696004.983541:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787696004.983542:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787696004.983545:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787696004.983546:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696004.983549:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787696004.983553:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545c00. 02000000:00000001:2.0:1787696004.983553:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787696004.983558:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696004.983563:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446667800 x1874531139560448/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:2.0:1787696004.983565:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445661000. 02000000:00000001:2.0:1787696004.983570:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696004.983577:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696004.983578:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787696004.983582:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1787696004.983582:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bc08. 00000400:00000010:2.0:1787696004.983590:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8b28. 00000100:00000040:1.0:1787696004.983591:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000009df211f4] to pc [ptlrpcd_rcv+-1] req@ffff8b1446667800 x1874531139560448/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787696004.983596:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139560192, portal 10 00000100:00000001:2.0:1787696004.983601:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787696004.983604:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929553920 : -128555779997696 : ffff8b1446664000) 00000100:00000001:1.0:1787696004.983607:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787696004.983612:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446664000 x1874531139560192/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787696004.983616:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787696004.983621:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787696004.983624:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787696004.983627:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787696004.983632:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787696004.983634:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1787696004.983638:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3892 00000100:00000040:2.0:1787696004.983640:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787696004.983645:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f86e8. 00000400:00000200:1.0:1787696004.983647:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000200:2.0:1787696004.983649:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139560192, offset 0 00000400:00000200:1.0:1787696004.983654:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787696004.983658:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000400:00000200:2.0:1787696004.983659:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:1.0:1787696004.983665:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:2.0:1787696004.983668:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696004.983671:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1787696004.983676:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000040:1.0:1787696004.983678:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:2.0:1787696004.983681:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1787696004.983682:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787696004.983687:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000200:2.0:1787696004.983687:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000020:00000040:1.0:1787696004.983690:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000400:00000200:2.0:1787696004.983693:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000020:00000001:1.0:1787696004.983695:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:2.0:1787696004.983698:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000020:00000040:1.0:1787696004.983699:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000400:00000200:2.0:1787696004.983704:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17186:17186:254:4294967295] 192.168.201.140@tcp LPNI seq info [17185:17185:6:4294967295] 00000020:00000001:1.0:1787696004.983705:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000020:00000001:1.0:1787696004.983710:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000200:2.0:1787696004.983715:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:1.0:1787696004.983716:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696004.983721:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787696004.983726:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696004.983730:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:2.0:1787696004.983731:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000020:00000040:1.0:1787696004.983733:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:1.0:1787696004.983740:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000800:00000200:2.0:1787696004.983742:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787696004.983744:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696004.983748:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000010:2.0:1787696004.983748:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5c00. 02000000:00000001:1.0:1787696004.983752:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:2.0:1787696004.983757:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000001:1.0:1787696004.983758:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696004.983763:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000200:2.1:1787696004.983765:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:2.0:1787696004.983769:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1787696004.983772:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000040:2.0:1787696004.983773:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000008:00000001:1.0:1787696004.983779:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696004.983783:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696004.983784:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696004.983788:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298565446 00000100:00000010:1.0:1787696004.983790:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665180. 00000100:00000001:2.0:1787696004.983792:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298576696 00000020:00000040:1.0:1787696004.983794:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000100:00000001:2.0:1787696004.983796:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298565446 02000000:00000001:1.0:1787696004.983800:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787696004.983800:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298576696 02000000:00000001:1.0:1787696004.983803:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696004.983804:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787696004.983809:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696004.983810:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787696004.983812:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696004.983813:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787696004.983817:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696004.983817:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696004.983824:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000010:1.0:1787696004.983825:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545400. 00000100:00000001:1.0:1787696004.983829:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696004.983835:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00100000:1.0:1787696004.983836:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446665180 x1874531139560576/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787696004.983840:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787696004.983843:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787696004.983846:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787696004.983849:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787696004.983854:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787696004.983854:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446666680 x1874531139560320/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787696004.983863:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f923372e] to pc [ptlrpcd_rcv+-1] req@ffff8b1446665180 x1874531139560576/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696004.983864:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787696004.983867:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696004.983870:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787696004.983872:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787696004.983876:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696004.983883:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787696004.983883:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446666680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139560320:192.168.201.140@tcp:8:kworker.0 00000100:00080000:1.0:1787696004.983889:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787696004.983890:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787696004.983893:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1787696004.983895:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:2.0:1787696004.983897:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696004.983898:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:2.0:1787696004.983901:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:1.0:1787696004.983902:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000010:2.0:1787696004.983906:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445663000. 00000100:00000001:1.0:1787696004.983907:0:38:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:2.0:1787696004.983911:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787696004.983913:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3892 00000400:00000200:2.0:1787696004.983918:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696004.983922:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787696004.983922:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752b160. 00000400:00000200:1.0:1787696004.983928:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000010:2.0:1787696004.983928:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f87f8. 00000100:00080000:1.0:1787696004.983932:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000200:2.0:1787696004.983934:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139560320, portal 4 00000100:00000001:2.0:1787696004.983938:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000040:1.0:1787696004.983939:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696004.983941:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929563776 : -128555779987840 : ffff8b1446666680) 00000100:00000001:1.0:1787696004.983944:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787696004.983948:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000040:2.0:1787696004.983951:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446666680 x1874531139560320/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696004.983958:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787696004.983963:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1787696004.983963:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:1.0:1787696004.983967:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000100:00000040:2.0:1787696004.983968:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000020:00000001:1.0:1787696004.983970:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000010:2.0:1787696004.983973:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8f68. 00000020:00000040:1.0:1787696004.983974:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:1.0:1787696004.983979:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000200:2.0:1787696004.983979:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139560320, offset 0 00000020:00000001:1.0:1787696004.983984:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000400:00000200:2.0:1787696004.983987:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:1.0:1787696004.983991:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:2.0:1787696004.983995:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696004.983996:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1787696004.984002:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000040:1.0:1787696004.984003:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696004.984005:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787696004.984006:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00080000:1.0:1787696004.984010:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787696004.984014:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696004.984015:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000001:1.0:1787696004.984017:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1787696004.984021:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 02000000:00000001:1.0:1787696004.984023:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:2.0:1787696004.984025:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 02000000:00000001:1.0:1787696004.984028:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696004.984031:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:1.0:1787696004.984032:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787696004.984037:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000200:2.0:1787696004.984037:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17187:17187:253:4294967295] 192.168.201.140@tcp LPNI seq info [17186:17186:5:4294967295] 00000008:00000001:1.0:1787696004.984043:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696004.984046:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696004.984047:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000010:1.0:1787696004.984051:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446664700. 00000020:00000040:1.0:1787696004.984056:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000400:00000200:2.0:1787696004.984061:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 02000000:00000001:1.0:1787696004.984062:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787696004.984065:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696004.984069:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787696004.984072:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000200:2.0:1787696004.984072:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 02000000:00000001:1.0:1787696004.984076:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1787696004.984079:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5200. 02000000:00000010:1.0:1787696004.984082:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446546800. 00000800:00000200:2.0:1787696004.984086:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:1.0:1787696004.984088:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.1:1787696004.984094:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00100000:1.0:1787696004.984098:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446664700 x1874531139560704/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787696004.984099:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787696004.984105:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:2.0:1787696004.984113:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787696004.984115:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298565476 00000100:00000001:2.0:1787696004.984117:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298576696 00000100:00000001:2.0:1787696004.984121:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298565476 00000100:00000001:2.0:1787696004.984127:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298576696 00000100:00000001:2.0:1787696004.984133:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696004.984134:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787696004.984136:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696004.984138:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696004.984140:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787696004.984145:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000007106cb5] to pc [ptlrpcd_rcv+-1] req@ffff8b1446664700 x1874531139560704/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787696004.984148:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446667800 x1874531139560448/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696004.984166:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696004.984168:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1787696004.984170:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787696004.984171:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696004.984175:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:1.0:1787696004.984178:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 02000000:00000001:2.0:1787696004.984178:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1787696004.984184:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1787696004.984184:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787696004.984188:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446667800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139560448:192.168.201.140@tcp:8:kworker.0 00000100:00000040:1.0:1787696004.984191:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000001:2.0:1787696004.984195:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000040:1.0:1787696004.984199:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3902) 02000000:00000001:2.0:1787696004.984205:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787696004.984207:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696004.984209:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787696004.984212:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445661400. 02000000:00000001:2.0:1787696004.984214:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696004.984218:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787696004.984220:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752b5d8. 00000400:00000010:2.0:1787696004.984223:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8c38. 00000100:00000200:2.0:1787696004.984226:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139560448, portal 4 00000100:00000001:2.0:1787696004.984228:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787696004.984230:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929568256 : -128555779983360 : ffff8b1446667800) 00000100:00000040:2.0:1787696004.984236:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446667800 x1874531139560448/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696004.984242:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787696004.984245:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787696004.984247:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8d48. 00000100:00000200:2.0:1787696004.984250:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139560448, offset 0 00000400:00000200:2.0:1787696004.984254:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787696004.984258:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696004.984261:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787696004.984263:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787696004.984266:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787696004.984268:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787696004.984270:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787696004.984273:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17188:17188:252:4294967295] 192.168.201.140@tcp LPNI seq info [17187:17187:4:4294967295] 00000400:00000200:2.0:1787696004.984276:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696004.984281:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787696004.984285:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787696004.984288:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5000. 00000800:00000200:2.0:1787696004.984291:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787696004.984295:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:2.0:1787696004.984298:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787696004.984300:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:2.0:1787696004.984308:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787696004.984310:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298565148 00000100:00000001:2.0:1787696004.984312:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298576696 00000100:00000001:2.0:1787696004.984314:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298565148 00000100:00000001:2.0:1787696004.984316:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298576696 00000100:00000001:2.0:1787696004.984318:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787696004.984320:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696004.984322:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696004.984324:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696004.984327:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787696004.984329:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1787696004.984331:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787696004.984332:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787696004.984333:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787696004.984351:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665180 x1874531139560576/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696004.984358:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787696004.984359:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696004.984361:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787696004.984379:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787696004.984381:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787696004.984384:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139560576:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787696004.984388:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787696004.984391:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787696004.984392:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696004.984394:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787696004.984396:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445660c00. 02000000:00000001:2.0:1787696004.984398:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696004.984401:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787696004.984403:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752b528. 00000400:00000010:2.0:1787696004.984406:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f84c8. 00000100:00000200:2.0:1787696004.984408:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139560576, portal 10 00000100:00000001:2.0:1787696004.984412:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787696004.984413:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929558400 : -128555779993216 : ffff8b1446665180) 00000100:00000040:2.0:1787696004.984418:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665180 x1874531139560576/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696004.984422:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787696004.984423:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787696004.984424:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8dd0. 00000100:00000200:2.0:1787696004.984426:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139560576, offset 0 00000400:00000200:2.0:1787696004.984429:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787696004.984431:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696004.984433:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787696004.984435:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787696004.984437:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787696004.984438:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787696004.984440:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787696004.984441:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17189:17189:251:4294967295] 192.168.201.140@tcp LPNI seq info [17188:17188:3:4294967295] 00000400:00000200:2.0:1787696004.984445:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696004.984450:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787696004.984454:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787696004.984457:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5b00. 00000800:00000200:2.0:1787696004.984461:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787696004.984464:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:2.0:1787696004.984466:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787696004.984469:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:2.0:1787696004.984472:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787696004.984474:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298565335 00000100:00000001:2.0:1787696004.984476:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298576696 00000100:00000001:2.0:1787696004.984478:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298565335 00000100:00000001:2.0:1787696004.984480:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298576696 00000100:00000001:2.0:1787696004.984482:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787696004.984488:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696004.984490:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787696004.984494:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446664700 x1874531139560704/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696004.984500:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787696004.984500:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696004.984502:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787696004.984503:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787696004.984504:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787696004.984506:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446664700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139560704:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787696004.984508:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787696004.984510:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787696004.984510:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696004.984511:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787696004.984513:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445662000. 02000000:00000001:2.0:1787696004.984514:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696004.984516:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787696004.984518:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752b948. 00000400:00000010:2.0:1787696004.984521:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8088. 00000100:00000200:2.0:1787696004.984523:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139560704, portal 10 00000100:00000001:2.0:1787696004.984525:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787696004.984526:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929555712 : -128555779995904 : ffff8b1446664700) 00000100:00000040:2.0:1787696004.984531:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446664700 x1874531139560704/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696004.984537:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787696004.984539:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787696004.984541:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8550. 00000100:00000200:2.0:1787696004.984543:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139560704, offset 0 00000400:00000200:2.0:1787696004.984546:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787696004.984550:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696004.984553:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787696004.984556:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787696004.984563:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787696004.984565:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787696004.984567:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787696004.984570:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17190:17190:250:4294967295] 192.168.201.140@tcp LPNI seq info [17189:17189:2:4294967295] 00000400:00000200:2.0:1787696004.984574:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696004.984578:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787696004.984581:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787696004.984585:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5500. 00000800:00000200:2.0:1787696004.984588:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787696004.984592:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:2.0:1787696004.984594:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787696004.984598:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:2.0:1787696004.984601:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787696004.984603:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298565313 00000100:00000001:2.0:1787696004.984605:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298576696 00000100:00000001:2.0:1787696004.984607:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298565313 00000100:00000001:2.0:1787696004.984609:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298576696 00000100:00000001:2.0:1787696004.984611:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787696004.984613:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696004.984616:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696004.984618:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696004.984621:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787696004.984623:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1787696004.984625:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787696004.984626:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787696004.984628:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696004.984629:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696005.046481:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:3.0:1787696005.046487:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787696005.046490:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787696005.046495:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:3.0:1787696005.046498:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787696005.046502:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787696005.046504:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787696005.046506:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787696005.046509:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696005.686483:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696005.686489:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696005.686493:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696005.686494:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696005.686497:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696005.686498:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696005.686500:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696005.686501:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696005.686503:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696005.686504:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696005.686506:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696005.686507:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787696005.686558:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787696005.686561:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787696005.686564:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787696006.070499:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787696006.070511:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787696006.070513:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787696006.070521:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:0.0:1787696006.070525:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787696006.070530:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696006.070533:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787696006.070536:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696006.070539:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696006.710482:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696006.710506:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696006.710511:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696006.710512:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696006.710514:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696006.710517:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787696006.710519:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787696006.710526:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787696006.710528:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787696006.710543:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696006.710546:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787696006.710548:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696006.710549:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696006.710551:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696006.710554:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696006.710556:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787696006.710557:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787696006.710558:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787696006.710559:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787696006.710560:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696006.710561:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787696006.710561:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696006.710562:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696006.710563:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696006.710565:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696006.710566:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696006.710568:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696006.710569:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787696006.710569:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787696006.710575:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787696006.710576:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787696006.710577:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696006.710578:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787696006.710578:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696006.710579:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696006.710583:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787696006.710647:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787696006.710654:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787696006.710659:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787696007.094469:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787696007.094476:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787696007.094493:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787696007.094499:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:0.0:1787696007.094502:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787696007.094505:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696007.094508:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787696007.094510:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696007.094513:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696007.734477:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696007.734483:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696007.734488:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696007.734490:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696007.734493:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696007.734494:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696007.734498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696007.734500:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696007.734502:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696007.734504:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696007.734507:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696007.734508:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787696007.734583:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787696007.734587:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787696007.734590:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1787696008.054872:0:2392:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.201.140:988 00000400:00000100:0.0:1787696008.054952:0:2392:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.201.140@tcp at host 192.168.201.140 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000400:00000200:0.0:1787696008.054966:0:2392:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787696008.054971:0:2392:0:(router.c:1791:lnet_notify()) 192.168.201.40@tcp notifying 192.168.201.140@tcp: down 00000400:00000200:0.0:1787696008.054976:0:2392:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:0.0:1787696008.054980:0:2392:0:(router.c:1888:lnet_notify()) Old news 00000800:00000100:0.0:1787696008.054986:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:0.0:1787696008.054989:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5400. 00000400:00000200:0.0:1787696008.054995:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1787696008.054998:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1787696008.055004:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8e58 00000400:00000010:0.0:1787696008.055006:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8e58. 00000100:00000001:0.0:1787696008.055010:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787696008.055017:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b1446665880 x1874531139559936/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787696008.055039:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787696008.055042:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665880 x1874531139559936/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696029 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787696008.055047:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787696008.055048:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1787696008.055055:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:0.0:1787696008.055056:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5300. 00000400:00000200:0.0:1787696008.055058:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1787696008.055060:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1787696008.055062:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8440 00000400:00000010:0.0:1787696008.055078:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8440. 00000100:00000001:0.0:1787696008.055080:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787696008.055083:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b1446664a80 x1874531139560064/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787696008.055088:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787696008.055090:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446664a80 x1874531139560064/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696029 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787696008.055094:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787696008.055094:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1787696008.055097:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:0.0:1787696008.055098:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5c00. 00000400:00000200:0.0:1787696008.055100:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1787696008.055102:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1787696008.055103:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f86e8 00000400:00000010:0.0:1787696008.055104:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f86e8. 00000100:00000001:0.0:1787696008.055106:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787696008.055108:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b1446664000 x1874531139560192/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787696008.055113:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787696008.055115:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446664000 x1874531139560192/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696029 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787696008.055118:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787696008.055119:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1787696008.055121:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:0.0:1787696008.055122:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5200. 00000400:00000200:0.0:1787696008.055124:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1787696008.055125:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1787696008.055126:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8f68 00000400:00000010:0.0:1787696008.055127:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8f68. 00000100:00000001:0.0:1787696008.055129:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787696008.055131:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b1446666680 x1874531139560320/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787696008.055141:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787696008.055143:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446666680 x1874531139560320/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696029 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787696008.055147:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787696008.055148:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1787696008.055150:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:0.0:1787696008.055151:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5000. 00000400:00000200:0.0:1787696008.055153:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1787696008.055154:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1787696008.055156:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8d48 00000400:00000010:0.0:1787696008.055157:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8d48. 00000100:00000001:0.0:1787696008.055158:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787696008.055160:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b1446667800 x1874531139560448/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787696008.055163:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787696008.055165:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446667800 x1874531139560448/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696029 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787696008.055168:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787696008.055169:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1787696008.055172:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:0.0:1787696008.055173:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5b00. 00000400:00000200:0.0:1787696008.055175:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1787696008.055177:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1787696008.055178:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8dd0 00000400:00000010:0.0:1787696008.055179:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8dd0. 00000100:00000001:0.0:1787696008.055180:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787696008.055182:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b1446665180 x1874531139560576/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787696008.055186:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787696008.055187:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665180 x1874531139560576/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696029 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787696008.055194:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787696008.055195:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:0.0:1787696008.055198:0:2392:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.201.140@tcp 00000800:00000010:0.0:1787696008.055200:0:2392:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5500. 00000400:00000200:0.0:1787696008.055201:0:2392:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -113, hstatus = 8 00000400:00000200:0.0:1787696008.055203:0:2392:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_DROPPED 00000400:00000200:0.0:1787696008.055204:0:2392:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8550 00000400:00000010:0.0:1787696008.055209:0:2392:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8550. 00000100:00000001:0.0:1787696008.055210:0:2392:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1787696008.055213:0:2392:0:(events.c:39:request_out_callback()) @@@ type 5, status -113 req@ffff8b1446664700 x1874531139560704/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696029 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787696008.055217:0:2392:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1787696008.055218:0:2392:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446664700 x1874531139560704/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696029 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787696008.055222:0:2392:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787696008.055222:0:2392:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787696008.055275:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787696008.055279:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787696008.055282:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787696008.055301:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696004/real 1787696008] req@ffff8b1446665180 x1874531139560576/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787696008.055329:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787696008.055332:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787696008.055335:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752b528. 00000400:00000200:2.0:1787696008.055338:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f84c8 00000400:00000010:2.0:1787696008.055339:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f84c8. 00000100:00000001:2.0:1787696008.055342:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787696008.055346:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665180 x1874531139560576/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787696008.055355:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665180 x1874531139560576/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.055362:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787696008.055400:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.055402:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696008.055403:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787696008.055406:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446665180 x1874531139560576/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.055411:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787696008.055414:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665180 x1874531139560576/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.055419:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787696008.055421:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.055422:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696008.055422:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.055425:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787696008.055427:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787696008.055429:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787696008.055433:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787696008.055435:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787696008.055445:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665180 x1874531139560576/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787696008.055459:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139560576:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787696008.055464:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787696008.055467:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696004/real 1787696008] req@ffff8b1446664700 x1874531139560704/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787696008.055474:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787696008.055475:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787696008.055477:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752b948. 00000400:00000200:2.0:1787696008.055479:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8088 00000400:00000010:2.0:1787696008.055481:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8088. 00000100:00000001:2.0:1787696008.055482:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787696008.055485:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446664700 x1874531139560704/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787696008.055492:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446664700 x1874531139560704/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.055498:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787696008.055499:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.055501:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696008.055502:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787696008.055505:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446664700 x1874531139560704/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.055510:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787696008.055514:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446664700 x1874531139560704/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.055519:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787696008.055521:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.055526:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696008.055528:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.055529:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787696008.055531:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787696008.055533:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787696008.055536:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787696008.055538:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787696008.055541:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446664700 x1874531139560704/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787696008.055548:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446664700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139560704:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787696008.055551:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787696008.055554:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696004/real 1787696008] req@ffff8b1446666680 x1874531139560320/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787696008.055560:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787696008.055561:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787696008.055563:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752b160. 00000400:00000200:2.0:1787696008.055565:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f87f8 00000400:00000010:2.0:1787696008.055566:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f87f8. 00000100:00000001:2.0:1787696008.055568:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787696008.055570:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446666680 x1874531139560320/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787696008.055577:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446666680 x1874531139560320/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.055582:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787696008.055583:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.055584:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696008.055585:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787696008.055588:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446666680 x1874531139560320/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.055603:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787696008.055608:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446666680 x1874531139560320/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.055613:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787696008.055615:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.055617:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696008.055618:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.055620:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787696008.055621:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787696008.055623:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787696008.055626:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787696008.055628:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787696008.055631:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446666680 x1874531139560320/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787696008.055638:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446666680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139560320:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787696008.055642:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787696008.055646:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696004/real 1787696008] req@ffff8b1446667800 x1874531139560448/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787696008.055653:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787696008.055654:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787696008.055656:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752b5d8. 00000400:00000200:2.0:1787696008.055658:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8c38 00000400:00000010:2.0:1787696008.055659:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8c38. 00000100:00000001:2.0:1787696008.055661:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787696008.055664:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446667800 x1874531139560448/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787696008.055679:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446667800 x1874531139560448/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.055685:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787696008.055687:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.055688:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696008.055689:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787696008.055693:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446667800 x1874531139560448/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.055699:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787696008.055703:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446667800 x1874531139560448/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.055707:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787696008.055708:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.055709:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696008.055710:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.055711:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787696008.055712:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787696008.055713:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787696008.055715:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787696008.055716:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787696008.055718:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446667800 x1874531139560448/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787696008.055725:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446667800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139560448:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787696008.055728:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787696008.055731:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696004/real 1787696008] req@ffff8b1446664a80 x1874531139560064/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787696008.055750:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787696008.055751:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787696008.055753:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752b688. 00000400:00000200:2.0:1787696008.055755:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8bb0 00000400:00000010:2.0:1787696008.055757:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8bb0. 00000100:00000001:2.0:1787696008.055759:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787696008.055762:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446664a80 x1874531139560064/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787696008.055770:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446664a80 x1874531139560064/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.055777:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787696008.055778:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.055780:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696008.055781:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787696008.055785:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446664a80 x1874531139560064/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.055791:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787696008.055795:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446664a80 x1874531139560064/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.055801:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787696008.055803:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.055804:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696008.055805:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.055808:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787696008.055809:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787696008.055812:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787696008.055815:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787696008.055817:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787696008.055826:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446664a80 x1874531139560064/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787696008.055834:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446664a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139560064:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787696008.055838:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787696008.055842:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696004/real 1787696008] req@ffff8b1446664000 x1874531139560192/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787696008.055850:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787696008.055852:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787696008.055854:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bc08. 00000400:00000200:2.0:1787696008.055856:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8b28 00000400:00000010:2.0:1787696008.055857:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8b28. 00000100:00000001:2.0:1787696008.055859:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787696008.055863:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446664000 x1874531139560192/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787696008.055870:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446664000 x1874531139560192/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.055876:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787696008.055878:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.055879:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696008.055881:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787696008.055884:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446664000 x1874531139560192/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.055888:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787696008.055890:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446664000 x1874531139560192/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.055894:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787696008.055895:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.055901:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696008.055902:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.055903:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787696008.055905:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787696008.055906:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787696008.055909:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787696008.055910:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787696008.055912:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446664000 x1874531139560192/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787696008.055916:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446664000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139560192:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787696008.055918:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787696008.055921:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696004/real 1787696008] req@ffff8b1446665880 x1874531139559936/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787696008.055926:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787696008.055926:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787696008.055928:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787696008.055929:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787696008.055930:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787696008.055932:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787696008.055934:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665880 x1874531139559936/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787696008.055938:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665880 x1874531139559936/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.055941:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787696008.055942:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.055943:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696008.055944:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787696008.055947:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446665880 x1874531139559936/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.055962:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787696008.055966:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665880 x1874531139559936/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.055972:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787696008.055974:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.055975:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696008.055977:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.055978:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787696008.055980:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787696008.055982:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787696008.055985:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787696008.055988:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787696008.055991:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665880 x1874531139559936/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787696008.055998:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139559936:192.168.201.140@tcp:250:kworker.0 00000100:00000001:2.0:1787696008.056001:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.056003:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787696008.056005:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665180 x1874531139560576/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.056009:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787696008.056011:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787696008.056014:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445660c00. 02000000:00000001:2.0:1787696008.056017:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787696008.056019:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787696008.056020:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:2.0:1787696008.056022:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787696008.056024:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545400. 02000000:00000001:2.0:1787696008.056027:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787696008.056032:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787696008.056034:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665180. 00000100:00000001:2.0:1787696008.056036:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787696008.056037:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.056038:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787696008.056040:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446664700 x1874531139560704/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.056044:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787696008.056045:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787696008.056045:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445662000. 02000000:00000001:2.0:1787696008.056047:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787696008.056047:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787696008.056048:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:2.0:1787696008.056049:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787696008.056050:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446546800. 02000000:00000001:2.0:1787696008.056052:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787696008.056053:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787696008.056054:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446664700. 00000100:00000001:2.0:1787696008.056055:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787696008.056056:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.056056:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787696008.056058:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446666680 x1874531139560320/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.056061:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787696008.056062:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787696008.056063:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445663000. 02000000:00000001:2.0:1787696008.056065:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787696008.056065:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787696008.056066:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:2.0:1787696008.056067:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787696008.056068:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446547400. 02000000:00000001:2.0:1787696008.056070:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787696008.056070:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787696008.056071:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446666680. 00000100:00000001:2.0:1787696008.056073:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787696008.056073:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.056076:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787696008.056078:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446667800 x1874531139560448/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.056084:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787696008.056085:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787696008.056086:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445661400. 02000000:00000001:2.0:1787696008.056089:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787696008.056096:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787696008.056097:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:2.0:1787696008.056099:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787696008.056100:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545c00. 02000000:00000001:2.0:1787696008.056102:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787696008.056103:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787696008.056105:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446667800. 00000100:00000001:2.0:1787696008.056107:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787696008.056108:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.056110:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787696008.056113:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446664a80 x1874531139560064/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.056119:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787696008.056121:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787696008.056122:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445663400. 02000000:00000001:2.0:1787696008.056124:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787696008.056125:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787696008.056127:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:2.0:1787696008.056128:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787696008.056129:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545000. 02000000:00000001:2.0:1787696008.056131:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787696008.056132:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787696008.056133:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446664a80. 00000100:00000001:2.0:1787696008.056135:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787696008.056136:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.056137:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787696008.056140:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446664000 x1874531139560192/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.056155:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787696008.056156:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787696008.056158:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445661000. 02000000:00000001:2.0:1787696008.056160:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787696008.056161:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787696008.056163:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:2.0:1787696008.056164:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787696008.056165:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446544c00. 02000000:00000001:2.0:1787696008.056168:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787696008.056169:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787696008.056171:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446664000. 00000100:00000001:2.0:1787696008.056173:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787696008.056174:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.056175:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787696008.056179:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665880 x1874531139559936/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696029 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696008.056185:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787696008.056186:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787696008.056188:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445660400. 02000000:00000001:2.0:1787696008.056190:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787696008.056191:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787696008.056192:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:2.0:1787696008.056193:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787696008.056195:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545800. 02000000:00000001:2.0:1787696008.056197:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787696008.056198:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787696008.056199:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665880. 00000100:00000001:2.0:1787696008.056202:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787696008.056203:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.056205:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696008.056213:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787696008.056215:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696008.056217:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787696008.056219:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787696008.118466:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787696008.118474:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787696008.118476:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787696008.118481:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:0.0:1787696008.118484:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787696008.118487:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696008.118490:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787696008.118492:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696008.118495:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696008.758444:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696008.758452:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696008.758457:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696008.758458:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696008.758461:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696008.758462:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696008.758465:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696008.758466:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696008.758469:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696008.758470:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696008.758473:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696008.758474:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787696008.758560:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787696008.758566:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787696008.758571:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787696009.142496:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787696009.142505:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787696009.142507:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787696009.142513:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:0.0:1787696009.142517:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787696009.142541:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696009.142543:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787696009.142545:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696009.142548:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696009.782511:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696009.782519:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696009.782525:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696009.782526:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696009.782529:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696009.782530:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696009.782533:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696009.782533:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696009.782536:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696009.782537:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696009.782539:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696009.782540:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787696009.782605:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787696009.782610:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787696009.782613:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787696010.102537:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787696010.102545:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787696010.102548:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787696010.102551:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787696010.102554:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787696010.102559:0:38:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3897 00000400:00000200:1.0:1787696010.102566:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696010.102571:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787696010.102574:0:38:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787696010.102577:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696010.102580:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787696010.102591:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696010.102594:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787696010.102597:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787696010.102599:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:1.0:1787696010.102602:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787696010.102606:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:1.0:1787696010.102608:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:1.0:1787696010.102610:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:1.0:1787696010.102614:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696010.102616:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787696010.102618:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696010.102620:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787696010.102623:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:1.0:1787696010.102631:0:38:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787696010.102634:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696010.102637:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787696010.102640:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696010.102643:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696010.102646:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787696010.102648:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787696010.102655:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446667100. 00000020:00000040:1.0:1787696010.102658:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:1.0:1787696010.102662:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787696010.102664:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696010.102667:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787696010.102669:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787696010.102674:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787696010.102681:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446546000. 00000100:00000001:1.0:1787696010.102688:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696010.102698:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446667100 x1874531139560832/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787696010.102721:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787696010.102731:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696010.102737:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f24be523] to pc [ptlrpcd_rcv+-1] req@ffff8b1446667100 x1874531139560832/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696010.102759:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787696010.102762:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787696010.102765:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787696010.102766:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787696010.102768:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787696010.102772:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787696010.102775:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3897 00000400:00000200:1.0:1787696010.102779:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696010.102781:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787696010.102783:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787696010.102786:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696010.102788:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787696010.102790:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696010.102792:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787696010.102794:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787696010.102795:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:1.0:1787696010.102797:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787696010.102799:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:1.0:1787696010.102801:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:1.0:1787696010.102803:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:1.0:1787696010.102805:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696010.102807:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787696010.102809:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696010.102811:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787696010.102813:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:1.0:1787696010.102815:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787696010.102825:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696010.102826:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787696010.102838:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696010.102840:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696010.102841:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787696010.102846:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787696010.102849:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696010.102851:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787696010.102854:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665f80. 00000020:00000040:1.0:1787696010.102856:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:1.0:1787696010.102858:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787696010.102859:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696010.102861:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787696010.102862:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787696010.102864:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787696010.102868:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446547800. 00000100:00000001:1.0:1787696010.102872:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696010.102877:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446665f80 x1874531139560960/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787696010.102884:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787696010.102886:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696010.102889:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000a0a2be0e] to pc [ptlrpcd_rcv+-1] req@ffff8b1446665f80 x1874531139560960/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696010.102897:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787696010.102899:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787696010.102903:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787696010.102905:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787696010.102906:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787696010.102909:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787696010.102911:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3897 00000400:00000200:1.0:1787696010.102916:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696010.102928:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787696010.102930:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787696010.102933:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696010.102935:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787696010.102937:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696010.102938:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787696010.102940:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787696010.102942:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:1.0:1787696010.102943:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787696010.102945:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:1.0:1787696010.102947:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:1.0:1787696010.102949:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:1.0:1787696010.102952:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696010.102953:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787696010.102955:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696010.102957:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787696010.102960:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:1.0:1787696010.102962:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787696010.102964:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696010.102966:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787696010.102969:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696010.102971:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696010.102973:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787696010.102975:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787696010.102977:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696010.102979:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787696010.102982:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446667b80. 00000020:00000040:1.0:1787696010.102984:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:1.0:1787696010.102987:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787696010.102988:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696010.102990:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787696010.102996:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787696010.102998:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787696010.103002:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446547c00. 00000100:00000001:1.0:1787696010.103005:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696010.103010:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446667b80 x1874531139561088/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787696010.103016:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787696010.103019:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696010.103022:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006b2dcf49] to pc [ptlrpcd_rcv+-1] req@ffff8b1446667b80 x1874531139561088/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696010.103030:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787696010.103032:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787696010.103036:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787696010.103037:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787696010.103038:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787696010.103040:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787696010.103042:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787696010.103045:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3897 00000100:00000001:2.0:1787696010.103049:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000400:00000200:1.0:1787696010.103051:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787696010.103053:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000400:00000200:1.0:1787696010.103057:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787696010.103061:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787696010.103063:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446667100 x1874531139560832/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1787696010.103067:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696010.103072:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787696010.103077:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696010.103082:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:2.0:1787696010.103082:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787696010.103086:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787696010.103087:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787696010.103089:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:1.0:1787696010.103092:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:2.0:1787696010.103093:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:1.0:1787696010.103095:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 02000000:00000001:2.0:1787696010.103097:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1787696010.103099:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 02000000:00000001:2.0:1787696010.103103:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787696010.103104:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00100000:2.0:1787696010.103110:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446667100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139560832:192.168.201.140@tcp:250:kworker.0 00000100:00000001:2.0:1787696010.103117:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787696010.103121:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787696010.103123:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696010.103126:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787696010.103130:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445660400. 00000100:00000040:1.0:1787696010.103134:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:2.0:1787696010.103134:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696010.103140:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:2.0:1787696010.103143:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:1.0:1787696010.103144:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696010.103149:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000010:2.0:1787696010.103150:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000020:00000040:1.0:1787696010.103152:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:1.0:1787696010.103158:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000010:2.0:1787696010.103159:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000001:1.0:1787696010.103163:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696010.103166:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000200:2.0:1787696010.103169:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139560832, portal 25 02000000:00000001:1.0:1787696010.103170:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787696010.103174:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1787696010.103175:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696010.103177:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929566464 : -128555779985152 : ffff8b1446667100) 00000100:00000001:1.0:1787696010.103179:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787696010.103184:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000040:2.0:1787696010.103188:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446667100 x1874531139560832/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1787696010.103189:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696010.103194:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787696010.103200:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665500. 00000100:00000001:2.0:1787696010.103202:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:1.0:1787696010.103205:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000100:00000040:2.0:1787696010.103209:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 02000000:00000001:1.0:1787696010.103211:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787696010.103214:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1787696010.103215:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8b28. 00000100:00000001:1.0:1787696010.103217:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787696010.103220:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:2.0:1787696010.103220:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139560832, offset 0 02000000:00000001:1.0:1787696010.103223:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696010.103229:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 02000000:00000010:1.0:1787696010.103230:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446544000. 00000100:00000001:1.0:1787696010.103236:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696010.103243:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446665500 x1874531139561216/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1787696010.103243:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696010.103264:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:1.0:1787696010.103271:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:2.0:1787696010.103273:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:1.0:1787696010.103280:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696010.103281:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000040:1.0:1787696010.103287:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004ae1679a] to pc [ptlrpcd_rcv+-1] req@ffff8b1446665500 x1874531139561216/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696010.103298:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787696010.103301:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1787696010.103306:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787696010.103312:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00080000:1.0:1787696010.103314:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1787696010.103317:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:2.0:1787696010.103319:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17191:17191:256:4294967295] 192.168.201.140@tcp LPNI seq info [17190:17190:8:4294967295] 00000100:00080000:1.0:1787696010.103320:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787696010.103327:0:38:0:(import.c:485:import_select_connection()) Process entered 00000400:00000200:2.0:1787696010.103330:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:1.0:1787696010.103334:0:38:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3897 00000400:00000200:1.0:1787696010.103339:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696010.103348:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000400:00000200:2.0:1787696010.103350:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00080000:1.0:1787696010.103352:0:38:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787696010.103359:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000800:00000200:2.0:1787696010.103401:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:1.0:1787696010.103403:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000010:2.0:1787696010.103408:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5500. 00000100:00000040:1.0:1787696010.103409:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696010.103415:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787696010.103418:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787696010.103421:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000800:00000200:2.0:1787696010.103422:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000020:00000001:1.0:1787696010.103423:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000200:2.0:1787696010.103427:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000020:00000040:1.0:1787696010.103428:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:1.0:1787696010.103433:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:1.0:1787696010.103438:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000800:00000200:2.1:1787696010.103438:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787696010.103444:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000100:00000040:1.0:1787696010.103445:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696010.103449:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787696010.103452:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696010.103455:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000800:00000100:2.0:1787696010.103456:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000020:00000040:1.0:1787696010.103458:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000800:00000010:2.0:1787696010.103461:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5500. 00000100:00080000:1.0:1787696010.103462:0:38:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787696010.103466:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696010.103468:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:2.0:1787696010.103469:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 02000000:00000001:1.0:1787696010.103470:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696010.103474:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696010.103475:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787696010.103478:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000400:00000200:2.0:1787696010.103479:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787696010.103485:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8b28 00000008:00000001:1.0:1787696010.103486:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1787696010.103489:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8b28. 00000100:00000001:1.0:1787696010.103490:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787696010.103495:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446667480. 00000100:00000001:2.0:1787696010.103496:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000020:00000040:1.0:1787696010.103498:0:38:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:1.0:1787696010.103502:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000200:2.0:1787696010.103503:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446667100 x1874531139560832/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787696010.103504:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696010.103507:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787696010.103509:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787696010.103512:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787696010.103517:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446544400. 00000100:00000001:1.0:1787696010.103522:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696010.103524:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:1.0:1787696010.103528:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446667480 x1874531139561344/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000040:2.0:1787696010.103532:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446667100 x1874531139560832/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696035 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696010.103536:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787696010.103539:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696010.103546:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000390bc45a] to pc [ptlrpcd_rcv+-1] req@ffff8b1446667480 x1874531139561344/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.103548:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696010.103552:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787696010.103556:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787696010.103559:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787696010.103559:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787696010.103564:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787696010.103567:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000200:2.0:1787696010.103568:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:1.0:1787696010.103569:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787696010.103572:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298581817 00000100:00000001:1.0:1787696010.103577:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787696010.103579:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298576696 00000100:00080000:1.0:1787696010.103581:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3897 00000100:00000001:2.0:1787696010.103584:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298581817 00000400:00000200:1.0:1787696010.103589:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787696010.103590:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298576696 00000400:00000200:1.0:1787696010.103593:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:2.0:1787696010.103595:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1787696010.103596:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787696010.103599:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696010.103600:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787696010.103601:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696010.103607:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696010.103609:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:1.0:1787696010.103612:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1787696010.103612:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000040:1.0:1787696010.103616:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:1.0:1787696010.103617:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:1.0:1787696010.103619:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000020:00000001:1.0:1787696010.103621:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00100000:2.0:1787696010.103622:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696010/real 1787696010] req@ffff8b1446667100 x1874531139560832/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787696010.103623:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000040:1.0:1787696010.103626:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696010.103627:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787696010.103628:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696010.103631:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:1.0:1787696010.103635:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:1.0:1787696010.103639:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787696010.103644:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696010.103648:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787696010.103652:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:2.0:1787696010.103653:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1787696010.103654:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696010.103657:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000001:2.0:1787696010.103657:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00100000:1.0:1787696010.103661:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000010:2.0:1787696010.103662:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000008:00000001:1.0:1787696010.103663:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696010.103666:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1787696010.103669:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446666d80. 00000020:00000040:1.0:1787696010.103670:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:1.0:1787696010.103671:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787696010.103672:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696010.103673:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:2.0:1787696010.103673:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 02000000:00000001:1.0:1787696010.103675:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787696010.103678:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1787696010.103678:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 02000000:00000010:1.0:1787696010.103683:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446545800. 00000100:00000001:2.0:1787696010.103684:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787696010.103686:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696010.103692:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446666d80 x1874531139561472/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000200:2.0:1787696010.103692:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446667100 x1874531139560832/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787696010.103724:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446667100 x1874531139560832/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696010.103741:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787696010.103742:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787696010.103746:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696010.103746:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.103752:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696010.103754:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787696010.103755:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006aa82a36] to pc [ptlrpcd_rcv+-1] req@ffff8b1446666d80 x1874531139561472/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787696010.103761:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446667100 x1874531139560832/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696010.103765:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787696010.103770:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787696010.103773:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787696010.103774:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787696010.103777:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000040:2.0:1787696010.103778:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446667100 x1874531139560832/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787696010.103779:0:38:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787696010.103785:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787696010.103790:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3897 00000100:00000001:2.0:1787696010.103790:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787696010.103794:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1787696010.103797:0:38:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787696010.103797:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696010.103800:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1787696010.103803:0:38:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:2.0:1787696010.103804:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:1.0:1787696010.103806:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787696010.103807:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787696010.103811:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000040:1.0:1787696010.103812:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00080000:2.0:1787696010.103816:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787696010.103818:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1787696010.103821:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696010.103824:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1787696010.103826:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000040:2.0:1787696010.103830:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446667100 x1874531139560832/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787696010.103832:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:1.0:1787696010.103835:0:38:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000020:00000001:1.0:1787696010.103838:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:2.0:1787696010.103841:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446667100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139560832:192.168.201.140@tcp:250:kworker.0 00000020:00000040:1.0:1787696010.103843:0:38:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000100:00000001:2.0:1787696010.103847:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1787696010.103848:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:2.0:1787696010.103851:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665f80 x1874531139560960/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787696010.103853:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:2.0:1787696010.103860:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1787696010.103863:0:38:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696010.103863:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696010.103868:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1787696010.103869:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787696010.103872:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1787696010.103873:0:38:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 02000000:00000001:2.0:1787696010.103874:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696010.103876:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00100000:2.0:1787696010.103878:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139560960:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787696010.103881:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787696010.103882:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787696010.103884:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1787696010.103886:0:38:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 02000000:00000001:2.0:1787696010.103887:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787696010.103890:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445661000. 00000100:00080000:1.0:1787696010.103892:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000001:2.0:1787696010.103893:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696010.103897:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696010.103901:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:1.0:1787696010.103902:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:2.0:1787696010.103906:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 02000000:00000001:1.0:1787696010.103907:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000010:2.0:1787696010.103910:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 02000000:00000001:1.0:1787696010.103912:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1787696010.103914:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139560960, portal 10 00000100:00000001:1.0:1787696010.103916:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787696010.103917:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787696010.103919:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929561984 : -128555779989632 : ffff8b1446665f80) 00000008:00100000:1.0:1787696010.103921:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000040:2.0:1787696010.103925:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665f80 x1874531139560960/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1787696010.103926:0:38:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696010.103932:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696010.103936:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787696010.103939:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000010:1.0:1787696010.103940:0:38:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b1446665c00. 00000400:00000010:2.0:1787696010.103942:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8b28. 00000020:00000040:1.0:1787696010.103944:0:38:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000100:00000200:2.0:1787696010.103951:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139560960, offset 0 02000000:00000001:1.0:1787696010.103952:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:2.0:1787696010.103957:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 02000000:00000001:1.0:1787696010.103960:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696010.103962:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696010.103963:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787696010.103966:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:2.0:1787696010.103967:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 02000000:00000001:1.0:1787696010.103968:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696010.103971:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 02000000:00000010:1.0:1787696010.103974:0:38:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446544c00. 00000400:00000200:2.0:1787696010.103976:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:1.0:1787696010.103979:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696010.103982:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00100000:1.0:1787696010.103986:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b1446665c00 x1874531139561600/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:2.0:1787696010.103989:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787696010.103997:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17192:17192:256:4294967295] 192.168.201.140@tcp LPNI seq info [17191:17191:8:4294967295] 00000100:00000001:1.0:1787696010.103998:0:38:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787696010.104001:0:38:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696010.104007:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:1.0:1787696010.104008:0:38:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002d782cc4] to pc [ptlrpcd_rcv+-1] req@ffff8b1446665c00 x1874531139561600/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696010.104016:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696010.104020:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:2.0:1787696010.104023:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000040:1.0:1787696010.104024:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:1.0:1787696010.104029:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:2.0:1787696010.104034:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000040:1.0:1787696010.104035:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000800:00000010:2.0:1787696010.104038:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5500. 00000800:00000200:2.0:1787696010.104043:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000040:1.0:1787696010.104045:0:38:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3907) 00000800:00000200:2.0:1787696010.104046:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1787696010.104051:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787696010.104053:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1787696010.104056:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787696010.104057:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5500. 00000400:00000200:2.0:1787696010.104060:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787696010.104062:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787696010.104064:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8b28 00000400:00000010:2.0:1787696010.104066:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8b28. 00000100:00000001:2.0:1787696010.104068:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787696010.104071:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446665f80 x1874531139560960/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104077:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787696010.104078:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665f80 x1874531139560960/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696035 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104082:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696010.104083:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787696010.104085:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787696010.104092:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000400:00000200:2.0:1787696010.104095:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787696010.104096:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298571718 00000100:00000001:2.0:1787696010.104098:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298581817 00000100:00000001:2.0:1787696010.104099:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298571718 00000100:00000001:2.0:1787696010.104100:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298581817 00000100:00000001:2.0:1787696010.104102:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787696010.104103:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696010.104104:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787696010.104106:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696010/real 1787696010] req@ffff8b1446665f80 x1874531139560960/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787696010.104111:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787696010.104111:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787696010.104112:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787696010.104114:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787696010.104115:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787696010.104116:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787696010.104118:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665f80 x1874531139560960/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787696010.104122:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665f80 x1874531139560960/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104126:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787696010.104126:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.104127:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696010.104128:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787696010.104130:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446665f80 x1874531139560960/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104133:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787696010.104135:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665f80 x1874531139560960/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104142:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787696010.104144:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.104145:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696010.104146:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.104147:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787696010.104148:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787696010.104149:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787696010.104151:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787696010.104152:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787696010.104154:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665f80 x1874531139560960/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787696010.104158:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139560960:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787696010.104160:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787696010.104162:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446667b80 x1874531139561088/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104166:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787696010.104167:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696010.104168:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787696010.104169:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787696010.104170:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787696010.104172:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446667b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139561088:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787696010.104174:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787696010.104175:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787696010.104176:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696010.104177:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787696010.104179:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445663400. 02000000:00000001:2.0:1787696010.104180:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696010.104184:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787696010.104186:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787696010.104188:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787696010.104190:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139561088, portal 10 00000100:00000001:2.0:1787696010.104191:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787696010.104192:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929569152 : -128555779982464 : ffff8b1446667b80) 00000100:00000040:2.0:1787696010.104195:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446667b80 x1874531139561088/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104199:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787696010.104201:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787696010.104202:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8b28. 00000100:00000200:2.0:1787696010.104203:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139561088, offset 0 00000400:00000200:2.0:1787696010.104206:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787696010.104208:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696010.104210:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787696010.104211:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787696010.104213:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787696010.104215:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787696010.104216:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787696010.104218:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17193:17193:256:4294967295] 192.168.201.140@tcp LPNI seq info [17192:17192:8:4294967295] 00000400:00000200:2.0:1787696010.104220:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696010.104224:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787696010.104226:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787696010.104228:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5500. 00000800:00000200:2.0:1787696010.104230:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787696010.104232:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1787696010.104234:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787696010.104236:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1787696010.104238:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787696010.104239:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5500. 00000400:00000200:2.0:1787696010.104240:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787696010.104242:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787696010.104244:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8b28 00000400:00000010:2.0:1787696010.104260:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8b28. 00000100:00000001:2.0:1787696010.104263:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787696010.104267:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446667b80 x1874531139561088/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104278:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787696010.104281:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446667b80 x1874531139561088/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696035 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104286:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696010.104287:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787696010.104291:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787696010.104293:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:2.0:1787696010.104296:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787696010.104298:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298570566 00000100:00000001:2.0:1787696010.104300:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298581817 00000100:00000001:2.0:1787696010.104302:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298570566 00000100:00000001:2.0:1787696010.104304:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298581817 00000100:00000001:2.0:1787696010.104306:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787696010.104308:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696010.104309:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787696010.104313:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696010/real 1787696010] req@ffff8b1446667b80 x1874531139561088/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787696010.104320:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787696010.104321:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787696010.104323:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787696010.104325:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787696010.104326:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787696010.104328:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787696010.104331:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446667b80 x1874531139561088/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787696010.104339:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446667b80 x1874531139561088/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104352:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787696010.104353:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.104355:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696010.104356:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787696010.104359:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446667b80 x1874531139561088/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104382:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787696010.104386:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446667b80 x1874531139561088/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104392:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787696010.104394:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.104395:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696010.104396:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.104398:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787696010.104400:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787696010.104402:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787696010.104405:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787696010.104407:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787696010.104411:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446667b80 x1874531139561088/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787696010.104418:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446667b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139561088:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787696010.104422:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.104424:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787696010.104427:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446667100 x1874531139560832/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104431:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787696010.104433:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787696010.104439:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445660400. 02000000:00000001:2.0:1787696010.104440:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787696010.104442:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787696010.104443:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:2.0:1787696010.104444:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787696010.104446:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446546000. 02000000:00000001:2.0:1787696010.104450:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787696010.104452:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787696010.104454:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446667100. 00000100:00000001:2.0:1787696010.104457:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787696010.104458:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.104460:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787696010.104463:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665f80 x1874531139560960/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104469:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787696010.104470:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787696010.104471:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445661000. 02000000:00000001:2.0:1787696010.104474:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787696010.104475:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787696010.104476:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:2.0:1787696010.104478:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787696010.104479:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446547800. 02000000:00000001:2.0:1787696010.104482:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787696010.104483:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787696010.104484:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665f80. 00000100:00000001:2.0:1787696010.104486:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787696010.104487:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.104489:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787696010.104492:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446667b80 x1874531139561088/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104497:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787696010.104498:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787696010.104499:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445663400. 02000000:00000001:2.0:1787696010.104501:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787696010.104501:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787696010.104502:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:2.0:1787696010.104507:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787696010.104508:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446547c00. 02000000:00000001:2.0:1787696010.104509:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787696010.104510:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787696010.104511:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446667b80. 00000100:00000001:2.0:1787696010.104512:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787696010.104513:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.104514:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.104520:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787696010.104521:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696010.104522:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787696010.104523:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787696010.104524:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787696010.104527:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665500 x1874531139561216/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104531:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787696010.104532:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696010.104533:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787696010.104534:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787696010.104535:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787696010.104537:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139561216:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787696010.104539:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787696010.104540:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787696010.104541:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696010.104542:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787696010.104544:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445663400. 02000000:00000001:2.0:1787696010.104545:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696010.104547:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787696010.104549:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787696010.104551:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787696010.104553:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139561216, portal 4 00000100:00000001:2.0:1787696010.104555:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787696010.104555:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929559296 : -128555779992320 : ffff8b1446665500) 00000100:00000040:2.0:1787696010.104559:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665500 x1874531139561216/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104569:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787696010.104571:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787696010.104572:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8b28. 00000100:00000200:2.0:1787696010.104573:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139561216, offset 0 00000400:00000200:2.0:1787696010.104576:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787696010.104578:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696010.104580:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787696010.104583:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787696010.104584:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787696010.104586:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787696010.104587:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787696010.104589:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17194:17194:256:4294967295] 192.168.201.140@tcp LPNI seq info [17193:17193:8:4294967295] 00000400:00000200:2.0:1787696010.104591:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696010.104595:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787696010.104598:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787696010.104599:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5500. 00000800:00000200:2.0:1787696010.104602:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787696010.104603:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1787696010.104606:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787696010.104607:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1787696010.104610:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787696010.104611:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5500. 00000400:00000200:2.0:1787696010.104613:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787696010.104614:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787696010.104616:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8b28 00000400:00000010:2.0:1787696010.104617:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8b28. 00000100:00000001:2.0:1787696010.104619:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787696010.104621:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446665500 x1874531139561216/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104625:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787696010.104627:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665500 x1874531139561216/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696035 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104637:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696010.104638:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787696010.104640:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787696010.104642:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:2.0:1787696010.104644:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787696010.104645:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298570597 00000100:00000001:2.0:1787696010.104647:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298581817 00000100:00000001:2.0:1787696010.104648:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298570597 00000100:00000001:2.0:1787696010.104650:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298581817 00000100:00000001:2.0:1787696010.104651:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787696010.104652:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696010.104653:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787696010.104655:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696010/real 1787696010] req@ffff8b1446665500 x1874531139561216/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787696010.104659:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787696010.104660:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787696010.104661:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787696010.104662:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787696010.104663:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787696010.104664:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787696010.104666:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665500 x1874531139561216/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787696010.104670:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665500 x1874531139561216/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104674:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787696010.104675:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.104676:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696010.104676:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787696010.104678:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446665500 x1874531139561216/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104685:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787696010.104687:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665500 x1874531139561216/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104691:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787696010.104692:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.104693:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696010.104694:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.104695:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787696010.104695:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787696010.104697:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787696010.104699:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787696010.104700:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787696010.104702:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665500 x1874531139561216/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787696010.104706:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139561216:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787696010.104708:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787696010.104710:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446667480 x1874531139561344/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104713:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787696010.104714:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696010.104715:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787696010.104716:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787696010.104717:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787696010.104719:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446667480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139561344:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787696010.104720:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787696010.104721:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787696010.104724:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696010.104725:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787696010.104727:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445661000. 02000000:00000001:2.0:1787696010.104728:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696010.104730:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787696010.104732:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787696010.104733:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787696010.104735:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139561344, portal 4 00000100:00000001:2.0:1787696010.104736:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787696010.104737:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929567360 : -128555779984256 : ffff8b1446667480) 00000100:00000040:2.0:1787696010.104740:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446667480 x1874531139561344/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104744:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787696010.104745:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787696010.104746:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8b28. 00000100:00000200:2.0:1787696010.104747:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139561344, offset 0 00000400:00000200:2.0:1787696010.104750:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787696010.104752:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696010.104753:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787696010.104755:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787696010.104756:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787696010.104758:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787696010.104759:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787696010.104761:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17195:17195:256:4294967295] 192.168.201.140@tcp LPNI seq info [17194:17194:8:4294967295] 00000400:00000200:2.0:1787696010.104763:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696010.104766:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787696010.104769:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787696010.104770:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5500. 00000800:00000200:2.0:1787696010.104772:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787696010.104773:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1787696010.104776:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787696010.104777:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1787696010.104779:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787696010.104784:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5500. 00000400:00000200:2.0:1787696010.104786:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787696010.104788:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787696010.104790:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8b28 00000400:00000010:2.0:1787696010.104791:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8b28. 00000100:00000001:2.0:1787696010.104792:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787696010.104794:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446667480 x1874531139561344/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104798:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787696010.104800:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446667480 x1874531139561344/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696035 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104803:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696010.104804:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787696010.104806:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787696010.104808:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:2.0:1787696010.104809:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787696010.104810:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298570269 00000100:00000001:2.0:1787696010.104812:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298581817 00000100:00000001:2.0:1787696010.104813:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298570269 00000100:00000001:2.0:1787696010.104814:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298581817 00000100:00000001:2.0:1787696010.104816:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787696010.104817:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696010.104818:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787696010.104820:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696010/real 1787696010] req@ffff8b1446667480 x1874531139561344/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787696010.104823:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787696010.104824:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787696010.104825:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787696010.104826:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787696010.104827:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787696010.104831:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787696010.104833:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446667480 x1874531139561344/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787696010.104837:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446667480 x1874531139561344/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104841:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787696010.104841:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.104842:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696010.104843:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787696010.104845:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446667480 x1874531139561344/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104848:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787696010.104850:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446667480 x1874531139561344/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104854:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787696010.104856:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.104856:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696010.104857:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.104858:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787696010.104859:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787696010.104860:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787696010.104862:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787696010.104863:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787696010.104865:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446667480 x1874531139561344/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787696010.104868:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446667480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139561344:192.168.201.140@tcp:8:kworker.0 00000100:00000001:2.0:1787696010.104870:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787696010.104872:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446666d80 x1874531139561472/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104878:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787696010.104879:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696010.104880:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787696010.104881:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787696010.104883:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787696010.104884:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446666d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139561472:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787696010.104886:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787696010.104887:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787696010.104888:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696010.104889:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787696010.104890:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445660400. 02000000:00000001:2.0:1787696010.104891:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696010.104893:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787696010.104894:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787696010.104896:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787696010.104898:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139561472, portal 10 00000100:00000001:2.0:1787696010.104899:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787696010.104900:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929565568 : -128555779986048 : ffff8b1446666d80) 00000100:00000040:2.0:1787696010.104903:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446666d80 x1874531139561472/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104907:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787696010.104908:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787696010.104909:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8b28. 00000100:00000200:2.0:1787696010.104910:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139561472, offset 0 00000400:00000200:2.0:1787696010.104912:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787696010.104914:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696010.104916:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787696010.104918:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787696010.104919:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787696010.104920:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787696010.104922:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787696010.104926:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17196:17196:256:4294967295] 192.168.201.140@tcp LPNI seq info [17195:17195:8:4294967295] 00000400:00000200:2.0:1787696010.104929:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696010.104932:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787696010.104935:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787696010.104937:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5500. 00000800:00000200:2.0:1787696010.104939:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787696010.104940:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1787696010.104943:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787696010.104946:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1787696010.104948:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787696010.104950:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5500. 00000400:00000200:2.0:1787696010.104951:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787696010.104953:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787696010.104955:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8b28 00000400:00000010:2.0:1787696010.104956:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8b28. 00000100:00000001:2.0:1787696010.104957:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787696010.104959:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446666d80 x1874531139561472/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104964:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787696010.104966:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446666d80 x1874531139561472/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696035 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.104970:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696010.104970:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787696010.104973:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787696010.104975:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:2.0:1787696010.104977:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787696010.104979:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298570455 00000100:00000001:2.0:1787696010.104981:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298581817 00000100:00000001:2.0:1787696010.104983:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298570455 00000100:00000001:2.0:1787696010.104985:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298581817 00000100:00000001:2.0:1787696010.104987:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787696010.104993:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696010.104995:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787696010.104999:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696010/real 1787696010] req@ffff8b1446666d80 x1874531139561472/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787696010.105007:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787696010.105008:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787696010.105010:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787696010.105012:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787696010.105014:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787696010.105016:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787696010.105019:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446666d80 x1874531139561472/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787696010.105026:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446666d80 x1874531139561472/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.105030:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787696010.105031:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.105032:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696010.105032:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787696010.105034:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446666d80 x1874531139561472/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.105038:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787696010.105040:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446666d80 x1874531139561472/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.105044:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787696010.105045:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.105046:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696010.105047:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.105048:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787696010.105049:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787696010.105050:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787696010.105056:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787696010.105057:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787696010.105060:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446666d80 x1874531139561472/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787696010.105064:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446666d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139561472:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787696010.105067:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1787696010.105068:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b1446665c00 x1874531139561600/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.105072:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787696010.105072:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696010.105073:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787696010.105074:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787696010.105076:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787696010.105077:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b1446665c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139561600:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787696010.105079:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787696010.105080:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787696010.105080:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696010.105081:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1787696010.105083:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1445661400. 02000000:00000001:2.0:1787696010.105085:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1787696010.105087:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:2.0:1787696010.105088:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b144752bbb0. 00000400:00000010:2.0:1787696010.105090:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8220. 00000100:00000200:2.0:1787696010.105092:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139561600, portal 10 00000100:00000001:2.0:1787696010.105093:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787696010.105094:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517929561088 : -128555779990528 : ffff8b1446665c00) 00000100:00000040:2.0:1787696010.105097:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b1446665c00 x1874531139561600/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.105101:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787696010.105106:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:2.0:1787696010.105107:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b14482f8b28. 00000100:00000200:2.0:1787696010.105109:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139561600, offset 0 00000400:00000200:2.0:1787696010.105111:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:2.0:1787696010.105114:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696010.105116:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:2.0:1787696010.105117:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1787696010.105119:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:2.0:1787696010.105120:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1787696010.105122:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:2.0:1787696010.105124:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17197:17197:256:4294967295] 192.168.201.140@tcp LPNI seq info [17196:17196:8:4294967295] 00000400:00000200:2.0:1787696010.105126:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696010.105129:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:2.0:1787696010.105132:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787696010.105134:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14517d5500. 00000800:00000200:2.0:1787696010.105136:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.0:1787696010.105138:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000200:2.1:1787696010.105140:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:2.1:1787696010.105141:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 58 secs later) 00000800:00000100:2.0:1787696010.105144:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:2.0:1787696010.105145:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14517d5500. 00000400:00000200:2.0:1787696010.105146:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:2.0:1787696010.105148:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:2.0:1787696010.105150:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8b28 00000400:00000010:2.0:1787696010.105151:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8b28. 00000100:00000001:2.0:1787696010.105152:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:2.0:1787696010.105154:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b1446665c00 x1874531139561600/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696035 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.105158:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787696010.105160:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b1446665c00 x1874531139561600/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696035 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.105164:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696010.105165:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787696010.105171:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787696010.105173:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:2.0:1787696010.105175:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:2.0:1787696010.105177:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298570433 00000100:00000001:2.0:1787696010.105178:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298581817 00000100:00000001:2.0:1787696010.105179:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298570433 00000100:00000001:2.0:1787696010.105181:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298581817 00000100:00000001:2.0:1787696010.105182:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787696010.105183:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696010.105184:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787696010.105186:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696010/real 1787696010] req@ffff8b1446665c00 x1874531139561600/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787696010.105190:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787696010.105191:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1787696010.105192:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b144752bbb0. 00000400:00000200:2.0:1787696010.105193:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b14482f8220 00000400:00000010:2.0:1787696010.105194:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b14482f8220. 00000100:00000001:2.0:1787696010.105195:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1787696010.105197:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b1446665c00 x1874531139561600/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1787696010.105201:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b1446665c00 x1874531139561600/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.105205:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787696010.105206:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.105207:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696010.105208:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787696010.105210:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b1446665c00 x1874531139561600/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.105213:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1787696010.105215:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b1446665c00 x1874531139561600/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.105222:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787696010.105223:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.105224:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696010.105225:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.105226:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787696010.105226:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787696010.105228:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787696010.105229:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:2.0:1787696010.105230:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1787696010.105233:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b1446665c00 x1874531139561600/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787696010.105237:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b1446665c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139561600:192.168.201.140@tcp:38:kworker.0 00000100:00000001:2.0:1787696010.105239:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.105240:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787696010.105242:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665500 x1874531139561216/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.105245:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787696010.105246:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787696010.105259:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445663400. 02000000:00000001:2.0:1787696010.105261:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787696010.105262:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787696010.105263:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:2.0:1787696010.105264:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787696010.105265:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446544000. 02000000:00000001:2.0:1787696010.105268:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787696010.105268:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787696010.105269:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665500. 00000100:00000001:2.0:1787696010.105271:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787696010.105272:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.105276:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787696010.105278:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446667480 x1874531139561344/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.105282:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787696010.105283:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787696010.105284:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445661000. 02000000:00000001:2.0:1787696010.105285:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787696010.105286:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787696010.105287:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:2.0:1787696010.105288:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787696010.105289:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446544400. 02000000:00000001:2.0:1787696010.105290:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787696010.105291:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787696010.105292:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446667480. 00000100:00000001:2.0:1787696010.105294:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787696010.105295:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.105296:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787696010.105299:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446666d80 x1874531139561472/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.105305:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787696010.105306:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787696010.105307:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445660400. 02000000:00000001:2.0:1787696010.105309:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787696010.105310:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787696010.105311:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:2.0:1787696010.105313:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787696010.105314:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446545800. 02000000:00000001:2.0:1787696010.105317:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787696010.105318:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787696010.105319:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446666d80. 00000100:00000001:2.0:1787696010.105321:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787696010.105323:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.105324:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1787696010.105327:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b1446665c00 x1874531139561600/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696035 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696010.105341:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787696010.105342:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1787696010.105343:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1445661400. 02000000:00000001:2.0:1787696010.105344:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787696010.105345:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:2.0:1787696010.105346:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:2.0:1787696010.105347:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:2.0:1787696010.105348:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446544c00. 02000000:00000001:2.0:1787696010.105349:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787696010.105350:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1787696010.105351:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b1446665c00. 00000100:00000001:2.0:1787696010.105352:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787696010.105353:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.105354:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696010.105356:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787696010.105357:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696010.105358:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787696010.105359:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787696010.166468:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787696010.166475:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787696010.166478:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787696010.166482:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:0.0:1787696010.166485:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787696010.166489:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696010.166492:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787696010.166494:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696010.166497:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696010.806461:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696010.806466:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696010.806470:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696010.806471:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696010.806474:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696010.806474:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696010.806477:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696010.806478:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696010.806480:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696010.806481:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696010.806482:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696010.806483:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787696010.806533:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787696010.806537:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787696010.806539:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787696011.190460:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787696011.190466:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787696011.190468:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787696011.190472:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:0.0:1787696011.190475:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787696011.190478:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696011.190481:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787696011.190483:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696011.190484:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696011.830455:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696011.830461:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696011.830468:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696011.830469:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696011.830472:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696011.830474:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696011.830478:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696011.830479:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696011.830482:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696011.830485:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787696011.830488:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787696011.830494:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787696011.830497:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787696011.830498:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696011.830502:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787696011.830504:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696011.830506:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696011.830509:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696011.830512:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696011.830514:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787696011.830567:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787696011.830570:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787696011.830573:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787696012.214465:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787696012.214477:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787696012.214480:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787696012.214488:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:0.0:1787696012.214493:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787696012.214499:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696012.214502:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787696012.214505:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696012.214509:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696012.854473:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696012.854478:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696012.854483:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696012.854484:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696012.854486:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696012.854503:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696012.854506:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696012.854507:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696012.854509:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696012.854510:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696012.854513:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696012.854514:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787696012.854618:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787696012.854641:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787696012.854646:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787696013.238460:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787696013.238467:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787696013.238469:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787696013.238473:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:0.0:1787696013.238476:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787696013.238479:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696013.238481:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787696013.238483:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696013.238485:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696013.878431:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696013.878436:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696013.878440:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696013.878441:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696013.878444:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696013.878445:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696013.878448:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696013.878448:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696013.878451:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696013.878452:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696013.878455:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696013.878456:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787696013.878497:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787696013.878501:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787696013.878503:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787696014.262511:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787696014.262520:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787696014.262522:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787696014.262528:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:0.0:1787696014.262531:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787696014.262536:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696014.262538:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787696014.262541:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696014.262543:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696014.902506:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696014.902516:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787696014.902518:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787696014.902521:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696014.902527:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787696014.902529:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696014.902531:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696014.902532:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696014.902536:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696014.902538:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787696014.902539:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787696014.902540:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696014.902541:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787696014.902542:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696014.902543:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696014.902545:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696014.902548:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696014.902549:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696014.902552:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696014.902576:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696014.902578:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696014.902579:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696014.902582:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696014.902583:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787696014.902653:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787696014.902659:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787696014.902663:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787696015.222777:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787696015.222791:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787696015.222795:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787696015.222800:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787696015.222806:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787696015.222816:0:47074:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3902 00000400:00000200:2.0:1787696015.222829:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696015.222839:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787696015.222842:0:47074:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787696015.222846:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.222849:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787696015.222851:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.222858:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787696015.222862:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787696015.222880:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:2.0:1787696015.222884:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787696015.222889:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:2.0:1787696015.222892:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:2.0:1787696015.222895:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:2.0:1787696015.222900:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.222902:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787696015.222905:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.222907:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787696015.222912:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:2.0:1787696015.222916:0:47074:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787696015.222920:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696015.222924:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787696015.222928:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787696015.222930:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696015.222932:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787696015.222936:0:47074:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787696015.222943:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465adc00. 00000020:00000040:2.0:1787696015.222947:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:2.0:1787696015.222951:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787696015.222953:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696015.222956:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787696015.222960:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787696015.222964:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787696015.222972:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661400. 00000100:00000001:2.0:1787696015.222980:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787696015.222990:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465adc00 x1874531139561728/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787696015.223009:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787696015.223014:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787696015.223020:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000114811d4] to pc [ptlrpcd_rcv+-1] req@ffff8b14465adc00 x1874531139561728/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696015.223061:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787696015.223065:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787696015.223069:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787696015.223071:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787696015.223073:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787696015.223076:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787696015.223084:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3902 00000400:00000200:2.0:1787696015.223089:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696015.223093:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787696015.223095:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787696015.223099:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.223101:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787696015.223103:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.223106:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787696015.223109:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787696015.223111:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:2.0:1787696015.223114:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787696015.223117:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:2.0:1787696015.223120:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:2.0:1787696015.223123:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000001:1.0:1787696015.223126:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1787696015.223128:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.223133:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787696015.223134:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1787696015.223136:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696015.223139:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787696015.223140:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787696015.223144:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00000040:1.0:1787696015.223150:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465adc00 x1874531139561728/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787696015.223150:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787696015.223156:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696015.223179:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787696015.223185:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787696015.223189:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696015.223193:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787696015.223198:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:2.0:1787696015.223200:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787696015.223202:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787696015.223203:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696015.223208:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696015.223210:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:2.0:1787696015.223211:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ad180. 02000000:00000001:1.0:1787696015.223215:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1787696015.223215:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:2.0:1787696015.223217:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787696015.223220:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696015.223222:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787696015.223224:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787696015.223226:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696015.223228:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787696015.223229:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445660400. 00000100:00000001:2.0:1787696015.223236:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696015.223237:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139561728:192.168.201.140@tcp:250:kworker.0 00000100:00100000:2.0:1787696015.223242:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ad180 x1874531139561856/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787696015.223254:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787696015.223255:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787696015.223257:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696015.223262:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787696015.223266:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696015.223270:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:2.0:1787696015.223274:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006ad773a4] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ad180 x1874531139561856/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1787696015.223278:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545c00. 02000000:00000001:1.0:1787696015.223284:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696015.223292:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696015.223296:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00080000:2.0:1787696015.223298:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:1.0:1787696015.223302:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000100:00080000:2.0:1787696015.223304:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787696015.223308:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787696015.223310:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000010:1.0:1787696015.223313:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000001:2.0:1787696015.223315:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787696015.223320:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3902 00000100:00000200:1.0:1787696015.223324:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139561728, portal 25 00000400:00000200:2.0:1787696015.223325:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696015.223330:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787696015.223332:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928807424 : -128555780744192 : ffff8b14465adc00) 00000400:00000200:2.0:1787696015.223341:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000040:1.0:1787696015.223344:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465adc00 x1874531139561728/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787696015.223351:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787696015.223357:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1787696015.223398:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000040:1.0:1787696015.223399:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:2.0:1787696015.223405:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000010:1.0:1787696015.223407:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000040:2.0:1787696015.223411:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000200:1.0:1787696015.223415:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139561728, offset 0 00000100:00000001:2.0:1787696015.223416:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787696015.223419:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787696015.223421:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:2.0:1787696015.223425:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787696015.223429:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:2.0:1787696015.223433:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000400:00000200:1.0:1787696015.223435:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000020:00000001:2.0:1787696015.223439:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:2.0:1787696015.223445:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.223446:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787696015.223451:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:2.0:1787696015.223451:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.223453:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787696015.223456:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:2.0:1787696015.223458:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787696015.223461:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696015.223462:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 02000000:00000001:2.0:1787696015.223463:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787696015.223468:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787696015.223471:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696015.223473:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:2.0:1787696015.223474:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787696015.223477:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787696015.223479:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696015.223482:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:2.0:1787696015.223483:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787696015.223488:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465afb80. 00000400:00000200:1.0:1787696015.223491:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000020:00000040:2.0:1787696015.223492:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:2.0:1787696015.223497:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1787696015.223498:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 02000000:00000001:2.0:1787696015.223499:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696015.223502:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787696015.223505:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000200:1.0:1787696015.223507:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17198:17198:256:4294967295] 192.168.201.140@tcp LPNI seq info [17197:17197:8:4294967295] 02000000:00000001:2.0:1787696015.223508:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696015.223515:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000010:2.0:1787696015.223523:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445661000. 00000400:00000200:1.0:1787696015.223529:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:2.0:1787696015.223533:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787696015.223539:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00100000:2.0:1787696015.223543:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465afb80 x1874531139561984/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000010:1.0:1787696015.223547:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000100:00000001:2.0:1787696015.223558:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:1.0:1787696015.223561:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:2.0:1787696015.223563:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787696015.223568:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 52 secs later) 00000100:00000040:2.0:1787696015.223572:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b9acb62b] to pc [ptlrpcd_rcv+-1] req@ffff8b14465afb80 x1874531139561984/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.1:1787696015.223581:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787696015.223588:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 52 secs later) 00000100:00000001:2.0:1787696015.223593:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787696015.223599:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00080000:2.0:1787696015.223600:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000010:1.0:1787696015.223603:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000100:00080000:2.0:1787696015.223604:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787696015.223608:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787696015.223610:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000200:1.0:1787696015.223611:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:2.0:1787696015.223613:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787696015.223619:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3902 00000400:00000200:2.0:1787696015.223624:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696015.223628:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787696015.223631:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787696015.223634:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.223636:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787696015.223639:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.223641:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787696015.223643:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787696015.223645:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:2.0:1787696015.223647:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787696015.223649:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:2.0:1787696015.223651:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:2.0:1787696015.223654:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:2.0:1787696015.223657:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.223659:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787696015.223667:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.223669:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787696015.223673:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000400:00000200:1.0:1787696015.223678:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00080000:2.0:1787696015.223681:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787696015.223686:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696015.223688:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 02000000:00000001:2.0:1787696015.223689:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000010:1.0:1787696015.223694:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 02000000:00000001:2.0:1787696015.223695:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787696015.223700:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.223702:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787696015.223703:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787696015.223708:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000200:1.0:1787696015.223710:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465adc00 x1874531139561728/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1787696015.223712:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696015.223717:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787696015.223723:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ad500. 00000020:00000040:2.0:1787696015.223728:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:2.0:1787696015.223731:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787696015.223732:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1787696015.223733:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696015.223737:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787696015.223740:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1787696015.223741:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465adc00 x1874531139561728/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696040 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787696015.223742:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787696015.223748:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663400. 00000100:00000001:2.0:1787696015.223753:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.223760:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787696015.223762:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ad500 x1874531139562112/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787696015.223763:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787696015.223771:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787696015.223772:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696015.223773:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787696015.223779:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004ab396cc] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ad500 x1874531139562112/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1787696015.223784:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696015.223789:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298586937 00000100:00000001:2.0:1787696015.223794:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787696015.223799:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787696015.223805:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1787696015.223806:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298581817 00000100:00000001:2.0:1787696015.223807:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787696015.223813:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298586937 00000100:00080000:2.0:1787696015.223818:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787696015.223820:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298581817 00000100:00000001:2.0:1787696015.223827:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787696015.223828:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696015.223833:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787696015.223834:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3902 00000100:00000001:1.0:1787696015.223839:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000400:00000200:2.0:1787696015.223843:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:2.0:1787696015.223848:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00100000:1.0:1787696015.223849:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696015/real 1787696015] req@ffff8b14465adc00 x1874531139561728/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787696015.223850:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:2.0:1787696015.223855:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.223860:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787696015.223865:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.223872:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000001:1.0:1787696015.223876:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1787696015.223876:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000001:1.0:1787696015.223880:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000040:2.0:1787696015.223881:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000400:00000010:1.0:1787696015.223885:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000020:00000001:2.0:1787696015.223886:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000200:1.0:1787696015.223891:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000020:00000040:2.0:1787696015.223891:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000400:00000010:1.0:1787696015.223895:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000020:00000001:2.0:1787696015.223897:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000001:1.0:1787696015.223901:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:2.0:1787696015.223902:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000200:1.0:1787696015.223907:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465adc00 x1874531139561728/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1787696015.223911:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.223918:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787696015.223922:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.223925:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787696015.223929:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00000200:1.0:1787696015.223930:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465adc00 x1874531139561728/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787696015.223934:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787696015.223939:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696015.223943:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787696015.223948:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696015.223949:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:2.0:1787696015.223951:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.223953:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696015.223954:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787696015.223958:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787696015.223959:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696015.223963:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1787696015.223963:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696015.223967:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696015.223972:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465adc00 x1874531139561728/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1787696015.223976:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465aca80. 00000020:00000040:2.0:1787696015.223981:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:2.0:1787696015.223986:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787696015.223989:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787696015.223989:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696015.223995:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787696015.223996:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000040:1.0:1787696015.224000:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465adc00 x1874531139561728/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787696015.224000:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1787696015.224003:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445663000. 00000100:00000001:2.0:1787696015.224009:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.224016:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:2.0:1787696015.224019:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465aca80 x1874531139562240/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787696015.224021:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.224031:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787696015.224033:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787696015.224035:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696015.224039:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.224040:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787696015.224044:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000040:2.0:1787696015.224047:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004fad31e4] to pc [ptlrpcd_rcv+-1] req@ffff8b14465aca80 x1874531139562240/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787696015.224049:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787696015.224059:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787696015.224065:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1787696015.224068:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787696015.224073:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1787696015.224074:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465adc00 x1874531139561728/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787696015.224079:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787696015.224083:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787696015.224087:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787696015.224094:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:1.0:1787696015.224096:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465adc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139561728:192.168.201.140@tcp:250:kworker.0 00000100:00080000:2.0:1787696015.224102:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3902 00000100:00000001:1.0:1787696015.224105:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.224109:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1787696015.224110:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:1.0:1787696015.224115:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465adc00 x1874531139561728/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1787696015.224120:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:2.0:1787696015.224127:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787696015.224132:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000040:2.0:1787696015.224133:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 02000000:00000001:1.0:1787696015.224136:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1787696015.224137:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000010:1.0:1787696015.224140:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545c00. 00000100:00000040:2.0:1787696015.224141:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.224146:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787696015.224148:0:47074:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1787696015.224150:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787696015.224153:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787696015.224154:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000040:2.0:1787696015.224156:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:1.0:1787696015.224171:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787696015.224175:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661400. 00000020:00000001:2.0:1787696015.224177:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1787696015.224180:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000020:00000040:2.0:1787696015.224181:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 02000000:00000001:1.0:1787696015.224185:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:2.0:1787696015.224186:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000010:1.0:1787696015.224190:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465adc00. 00000020:00000001:2.0:1787696015.224190:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000001:1.0:1787696015.224197:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000040:2.0:1787696015.224198:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696015.224202:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696015.224202:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787696015.224206:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696015.224208:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787696015.224209:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:2.0:1787696015.224214:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:2.0:1787696015.224217:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:2.0:1787696015.224218:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696015.224220:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787696015.224221:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1787696015.224222:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696015.224225:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787696015.224226:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696015.224232:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787696015.224233:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:2.0:1787696015.224234:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787696015.224236:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00000001:2.0:1787696015.224237:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.224240:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787696015.224241:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1787696015.224244:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465ad880. 00000020:00000040:2.0:1787696015.224247:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:2.0:1787696015.224253:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787696015.224256:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696015.224257:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ad180 x1874531139561856/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696015.224260:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787696015.224263:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787696015.224267:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.224274:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000010:2.0:1787696015.224276:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445662000. 00000100:00000001:1.0:1787696015.224277:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696015.224283:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787696015.224285:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696015.224286:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696015.224291:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696015.224297:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139561856:192.168.201.140@tcp:38:kworker.0 00000100:00100000:2.0:1787696015.224297:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465ad880 x1874531139562368/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787696015.224306:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787696015.224310:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787696015.224314:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696015.224314:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787696015.224319:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787696015.224319:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1787696015.224324:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545c00. 02000000:00000001:1.0:1787696015.224327:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696015.224337:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:2.0:1787696015.224338:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000075b4d632] to pc [ptlrpcd_rcv+-1] req@ffff8b14465ad880 x1874531139562368/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1787696015.224342:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787696015.224348:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787696015.224355:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139561856, portal 10 00000100:00000001:2.0:1787696015.224357:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.224359:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1787696015.224362:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787696015.224385:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928804736 : -128555780746880 : ffff8b14465ad180) 00000100:00080000:2.0:1787696015.224392:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787696015.224394:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787696015.224395:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000040:1.0:1787696015.224398:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ad180 x1874531139561856/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787696015.224399:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787696015.224405:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3902 00000400:00000200:2.0:1787696015.224415:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696015.224417:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:2.0:1787696015.224420:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000040:1.0:1787696015.224426:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00080000:2.0:1787696015.224426:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000400:00000010:1.0:1787696015.224432:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000040:2.0:1787696015.224433:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.224438:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000200:1.0:1787696015.224440:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139561856, offset 0 00000100:00000040:2.0:1787696015.224441:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.224444:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:2.0:1787696015.224447:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:2.0:1787696015.224449:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000400:00000200:1.0:1787696015.224451:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000020:00000001:2.0:1787696015.224452:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:2.0:1787696015.224456:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:2.0:1787696015.224459:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000400:00000200:1.0:1787696015.224460:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:2.0:1787696015.224462:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:2.0:1787696015.224468:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:1.0:1787696015.224470:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:2.0:1787696015.224471:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1787696015.224476:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.224479:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787696015.224480:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:2.0:1787696015.224481:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00080000:2.0:1787696015.224487:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:1.0:1787696015.224488:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00000001:2.0:1787696015.224490:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696015.224493:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 02000000:00000001:2.0:1787696015.224497:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787696015.224500:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787696015.224504:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696015.224506:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:2.0:1787696015.224507:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787696015.224511:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1787696015.224514:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17199:17199:256:4294967295] 192.168.201.140@tcp LPNI seq info [17198:17198:8:4294967295] 00000008:00000001:2.0:1787696015.224515:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696015.224519:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696015.224525:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000010:2.0:1787696015.224525:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14465af800. 00000020:00000040:2.0:1787696015.224530:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:2.0:1787696015.224533:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787696015.224536:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787696015.224539:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1787696015.224541:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 02000000:00000001:2.0:1787696015.224542:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787696015.224547:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787696015.224552:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 02000000:00000010:2.0:1787696015.224554:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1445660c00. 00000800:00000010:1.0:1787696015.224557:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000100:00000001:2.0:1787696015.224560:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787696015.224565:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787696015.224572:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 52 secs later) 00000100:00100000:2.0:1787696015.224573:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14465af800 x1874531139562496/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000200:1.1:1787696015.224584:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:2.0:1787696015.224589:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:1.1:1787696015.224594:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 52 secs later) 00000100:00000001:2.0:1787696015.224595:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787696015.224605:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b410802f] to pc [ptlrpcd_rcv+-1] req@ffff8b14465af800 x1874531139562496/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1787696015.224607:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696015.224612:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000100:00000001:2.0:1787696015.224621:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1787696015.224624:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787696015.224627:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:2.0:1787696015.224630:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000040:2.0:1787696015.224636:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000200:1.0:1787696015.224640:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00000040:2.0:1787696015.224646:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1787696015.224647:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787696015.224652:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787696015.224656:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:2.0:1787696015.224657:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3913) 00000100:00000200:1.0:1787696015.224662:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ad180 x1874531139561856/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.224675:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696015.224679:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ad180 x1874531139561856/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696040 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.224688:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.224690:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787696015.224694:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696015.224699:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000400:00000200:1.0:1787696015.224704:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696015.224707:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298576839 00000100:00000001:1.0:1787696015.224711:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298586937 00000100:00000001:1.0:1787696015.224713:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298576839 00000100:00000001:1.0:1787696015.224716:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298586937 00000100:00000001:1.0:1787696015.224723:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696015.224725:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.224727:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787696015.224732:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696015/real 1787696015] req@ffff8b14465ad180 x1874531139561856/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787696015.224742:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787696015.224743:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787696015.224745:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787696015.224748:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787696015.224750:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787696015.224752:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787696015.224756:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ad180 x1874531139561856/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787696015.224765:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ad180 x1874531139561856/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.224773:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787696015.224775:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.224777:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696015.224778:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787696015.224782:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ad180 x1874531139561856/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.224791:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787696015.224795:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ad180 x1874531139561856/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.224804:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787696015.224807:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.224808:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696015.224809:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.224811:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787696015.224813:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787696015.224819:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787696015.224823:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787696015.224826:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787696015.224830:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ad180 x1874531139561856/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787696015.224839:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139561856:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696015.224844:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787696015.224847:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465afb80 x1874531139561984/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.224855:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787696015.224856:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696015.224859:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787696015.224861:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696015.224863:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696015.224866:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139561984:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696015.224870:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787696015.224872:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787696015.224874:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696015.224876:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787696015.224879:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547400. 02000000:00000001:1.0:1787696015.224881:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696015.224885:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787696015.224888:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787696015.224891:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787696015.224894:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139561984, portal 10 00000100:00000001:1.0:1787696015.224896:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787696015.224898:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928815488 : -128555780736128 : ffff8b14465afb80) 00000100:00000040:1.0:1787696015.224904:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465afb80 x1874531139561984/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.224922:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787696015.224925:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787696015.224927:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787696015.224930:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139561984, offset 0 00000400:00000200:1.0:1787696015.224934:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787696015.224938:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696015.224942:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787696015.224945:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787696015.224948:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787696015.224951:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787696015.224954:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787696015.224957:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17200:17200:256:4294967295] 192.168.201.140@tcp LPNI seq info [17199:17199:8:4294967295] 00000400:00000200:1.0:1787696015.224962:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696015.224970:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787696015.224975:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696015.224978:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787696015.224982:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787696015.224985:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1787696015.224990:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787696015.224993:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 52 secs later) 00000800:00000100:1.0:1787696015.224998:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696015.224999:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787696015.225003:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787696015.225006:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787696015.225009:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787696015.225011:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787696015.225014:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787696015.225018:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465afb80 x1874531139561984/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225027:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696015.225030:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465afb80 x1874531139561984/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696040 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225049:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.225050:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787696015.225054:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696015.225057:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787696015.225061:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696015.225064:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298575687 00000100:00000001:1.0:1787696015.225066:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298586937 00000100:00000001:1.0:1787696015.225068:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298575687 00000100:00000001:1.0:1787696015.225071:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298586937 00000100:00000001:1.0:1787696015.225074:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696015.225076:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.225078:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787696015.225082:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696015/real 1787696015] req@ffff8b14465afb80 x1874531139561984/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787696015.225091:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787696015.225093:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787696015.225095:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787696015.225097:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787696015.225099:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787696015.225101:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787696015.225105:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465afb80 x1874531139561984/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787696015.225115:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465afb80 x1874531139561984/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225122:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787696015.225124:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.225126:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696015.225128:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787696015.225131:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465afb80 x1874531139561984/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225146:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787696015.225150:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465afb80 x1874531139561984/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225158:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787696015.225175:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.225177:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696015.225179:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.225181:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787696015.225183:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787696015.225186:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787696015.225190:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787696015.225193:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787696015.225197:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465afb80 x1874531139561984/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787696015.225207:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465afb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139561984:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696015.225212:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787696015.225216:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ad500 x1874531139562112/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225223:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787696015.225225:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696015.225228:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787696015.225230:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696015.225232:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696015.225235:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139562112:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787696015.225239:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787696015.225241:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787696015.225247:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696015.225249:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787696015.225252:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546800. 02000000:00000001:1.0:1787696015.225254:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696015.225259:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787696015.225262:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787696015.225265:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787696015.225269:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139562112, portal 4 00000100:00000001:1.0:1787696015.225271:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787696015.225272:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928805632 : -128555780745984 : ffff8b14465ad500) 00000100:00000040:1.0:1787696015.225279:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ad500 x1874531139562112/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225286:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787696015.225288:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787696015.225291:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787696015.225293:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139562112, offset 0 00000400:00000200:1.0:1787696015.225297:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787696015.225302:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696015.225305:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787696015.225309:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787696015.225311:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787696015.225314:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787696015.225317:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787696015.225320:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17201:17201:256:4294967295] 192.168.201.140@tcp LPNI seq info [17200:17200:8:4294967295] 00000400:00000200:1.0:1787696015.225325:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696015.225332:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787696015.225338:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696015.225341:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787696015.225345:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787696015.225348:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1787696015.225353:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787696015.225356:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 52 secs later) 00000800:00000100:1.0:1787696015.225361:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696015.225389:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787696015.225394:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787696015.225399:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787696015.225403:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787696015.225405:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787696015.225408:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787696015.225412:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ad500 x1874531139562112/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225422:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696015.225426:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ad500 x1874531139562112/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696040 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225433:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.225436:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787696015.225439:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696015.225442:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787696015.225446:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696015.225448:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298575718 00000100:00000001:1.0:1787696015.225451:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298586937 00000100:00000001:1.0:1787696015.225454:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298575718 00000100:00000001:1.0:1787696015.225456:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298586937 00000100:00000001:1.0:1787696015.225459:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696015.225461:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.225463:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787696015.225467:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696015/real 1787696015] req@ffff8b14465ad500 x1874531139562112/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787696015.225476:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787696015.225478:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787696015.225479:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787696015.225482:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787696015.225484:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787696015.225491:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787696015.225495:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ad500 x1874531139562112/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787696015.225504:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ad500 x1874531139562112/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225511:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787696015.225513:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.225515:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696015.225517:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787696015.225521:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ad500 x1874531139562112/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225528:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787696015.225532:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ad500 x1874531139562112/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225540:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787696015.225543:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.225544:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696015.225545:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.225547:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787696015.225549:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787696015.225551:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787696015.225554:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787696015.225557:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787696015.225561:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ad500 x1874531139562112/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787696015.225570:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139562112:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787696015.225574:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787696015.225578:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465aca80 x1874531139562240/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225589:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787696015.225591:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696015.225593:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787696015.225595:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696015.225597:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696015.225600:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139562240:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787696015.225604:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787696015.225606:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787696015.225608:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696015.225610:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787696015.225613:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787696015.225615:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696015.225619:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787696015.225621:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787696015.225625:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787696015.225628:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139562240, portal 4 00000100:00000001:1.0:1787696015.225630:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787696015.225632:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928802944 : -128555780748672 : ffff8b14465aca80) 00000100:00000040:1.0:1787696015.225638:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465aca80 x1874531139562240/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225646:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787696015.225648:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787696015.225650:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787696015.225653:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139562240, offset 0 00000400:00000200:1.0:1787696015.225657:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787696015.225660:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696015.225664:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787696015.225667:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787696015.225670:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787696015.225673:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787696015.225675:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787696015.225682:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17202:17202:256:4294967295] 192.168.201.140@tcp LPNI seq info [17201:17201:8:4294967295] 00000400:00000200:1.0:1787696015.225687:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696015.225695:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787696015.225700:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696015.225703:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787696015.225707:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787696015.225710:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1787696015.225715:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787696015.225718:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 52 secs later) 00000800:00000100:1.0:1787696015.225722:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696015.225724:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787696015.225727:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787696015.225731:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787696015.225734:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787696015.225736:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787696015.225739:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787696015.225743:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465aca80 x1874531139562240/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225751:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696015.225755:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465aca80 x1874531139562240/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696040 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225763:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.225765:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787696015.225767:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696015.225770:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787696015.225774:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696015.225776:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298575390 00000100:00000001:1.0:1787696015.225779:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298586937 00000100:00000001:1.0:1787696015.225782:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298575390 00000100:00000001:1.0:1787696015.225784:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298586937 00000100:00000001:1.0:1787696015.225786:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696015.225792:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.225795:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787696015.225798:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696015/real 1787696015] req@ffff8b14465aca80 x1874531139562240/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787696015.225808:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787696015.225810:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787696015.225812:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787696015.225814:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787696015.225816:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787696015.225818:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787696015.225821:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465aca80 x1874531139562240/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787696015.225831:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465aca80 x1874531139562240/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225838:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787696015.225840:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.225842:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696015.225843:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787696015.225847:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465aca80 x1874531139562240/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225855:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787696015.225859:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465aca80 x1874531139562240/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225867:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787696015.225869:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.225871:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696015.225872:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.225874:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787696015.225876:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787696015.225879:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787696015.225888:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787696015.225890:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787696015.225895:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465aca80 x1874531139562240/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787696015.225904:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465aca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139562240:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787696015.225908:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.225910:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696015.225914:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ad180 x1874531139561856/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225921:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787696015.225923:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787696015.225925:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545c00. 02000000:00000001:1.0:1787696015.225929:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787696015.225930:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787696015.225932:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787696015.225934:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787696015.225936:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445660400. 02000000:00000001:1.0:1787696015.225939:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787696015.225941:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787696015.225942:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ad180. 00000100:00000001:1.0:1787696015.225945:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787696015.225947:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.225948:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696015.225952:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465afb80 x1874531139561984/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225959:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787696015.225961:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787696015.225962:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547400. 02000000:00000001:1.0:1787696015.225966:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787696015.225967:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787696015.225968:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787696015.225970:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787696015.225976:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445661000. 02000000:00000001:1.0:1787696015.225979:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787696015.225980:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787696015.225982:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465afb80. 00000100:00000001:1.0:1787696015.225985:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787696015.225986:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.225988:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696015.225992:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ad500 x1874531139562112/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.225999:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787696015.226001:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787696015.226002:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546800. 02000000:00000001:1.0:1787696015.226006:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787696015.226008:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787696015.226009:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787696015.226011:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787696015.226013:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663400. 02000000:00000001:1.0:1787696015.226015:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787696015.226016:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787696015.226017:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ad500. 00000100:00000001:1.0:1787696015.226019:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787696015.226020:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.226021:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696015.226025:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465aca80 x1874531139562240/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.226032:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787696015.226034:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787696015.226035:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787696015.226037:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787696015.226038:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787696015.226040:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787696015.226042:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787696015.226043:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445663000. 02000000:00000001:1.0:1787696015.226045:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787696015.226047:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787696015.226048:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465aca80. 00000100:00000001:1.0:1787696015.226055:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787696015.226056:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.226059:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.226063:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787696015.226065:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.226067:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787696015.226069:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787696015.226070:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787696015.226074:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465ad880 x1874531139562368/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.226082:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787696015.226084:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696015.226086:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787696015.226088:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696015.226090:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696015.226094:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139562368:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696015.226097:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787696015.226099:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787696015.226101:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696015.226103:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787696015.226105:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787696015.226108:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696015.226112:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787696015.226114:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787696015.226118:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787696015.226121:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139562368, portal 10 00000100:00000001:1.0:1787696015.226123:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787696015.226124:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928806528 : -128555780745088 : ffff8b14465ad880) 00000100:00000040:1.0:1787696015.226131:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465ad880 x1874531139562368/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.226139:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787696015.226141:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787696015.226143:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787696015.226146:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139562368, offset 0 00000400:00000200:1.0:1787696015.226155:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787696015.226172:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696015.226177:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787696015.226180:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787696015.226183:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787696015.226185:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787696015.226188:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787696015.226192:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17203:17203:256:4294967295] 192.168.201.140@tcp LPNI seq info [17202:17202:8:4294967295] 00000400:00000200:1.0:1787696015.226196:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696015.226203:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787696015.226208:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696015.226212:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787696015.226216:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787696015.226219:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1787696015.226223:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787696015.226226:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 52 secs later) 00000800:00000100:1.0:1787696015.226231:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696015.226233:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787696015.226236:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787696015.226240:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787696015.226244:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787696015.226246:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787696015.226248:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787696015.226252:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465ad880 x1874531139562368/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.226261:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696015.226264:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465ad880 x1874531139562368/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696040 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.226272:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.226274:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787696015.226277:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696015.226280:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787696015.226284:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696015.226290:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298575576 00000100:00000001:1.0:1787696015.226293:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298586937 00000100:00000001:1.0:1787696015.226296:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298575576 00000100:00000001:1.0:1787696015.226298:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298586937 00000100:00000001:1.0:1787696015.226301:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696015.226303:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.226305:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787696015.226309:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696015/real 1787696015] req@ffff8b14465ad880 x1874531139562368/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787696015.226318:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787696015.226320:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787696015.226321:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787696015.226324:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787696015.226326:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787696015.226328:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787696015.226332:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465ad880 x1874531139562368/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787696015.226341:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465ad880 x1874531139562368/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.226349:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787696015.226351:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.226353:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696015.226355:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787696015.226358:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465ad880 x1874531139562368/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.226388:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787696015.226393:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465ad880 x1874531139562368/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.226401:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787696015.226409:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.226411:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696015.226412:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.226414:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787696015.226416:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787696015.226419:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787696015.226423:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787696015.226426:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787696015.226431:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465ad880 x1874531139562368/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787696015.226440:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465ad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139562368:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696015.226444:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787696015.226448:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14465af800 x1874531139562496/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.226456:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787696015.226458:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696015.226460:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787696015.226462:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696015.226464:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696015.226467:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14465af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139562496:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696015.226471:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787696015.226472:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787696015.226474:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696015.226476:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787696015.226479:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546800. 02000000:00000001:1.0:1787696015.226481:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696015.226484:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787696015.226487:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787696015.226491:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787696015.226494:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139562496, portal 10 00000100:00000001:1.0:1787696015.226500:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787696015.226502:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517928814592 : -128555780737024 : ffff8b14465af800) 00000100:00000040:1.0:1787696015.226508:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14465af800 x1874531139562496/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.226516:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787696015.226518:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787696015.226521:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787696015.226524:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139562496, offset 0 00000400:00000200:1.0:1787696015.226528:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787696015.226532:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696015.226535:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787696015.226538:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787696015.226541:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787696015.226544:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787696015.226546:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787696015.226550:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17204:17204:256:4294967295] 192.168.201.140@tcp LPNI seq info [17203:17203:8:4294967295] 00000400:00000200:1.0:1787696015.226554:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696015.226562:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787696015.226566:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696015.226570:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787696015.226574:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787696015.226577:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 52 secs later) 00000800:00000200:1.1:1787696015.226582:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787696015.226585:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 52 secs later) 00000800:00000100:1.0:1787696015.226589:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696015.226592:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787696015.226595:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787696015.226599:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787696015.226602:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787696015.226604:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787696015.226606:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787696015.226610:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14465af800 x1874531139562496/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696040 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.226631:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696015.226634:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14465af800 x1874531139562496/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696040 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.226642:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.226644:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787696015.226647:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696015.226650:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787696015.226654:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696015.226657:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298575555 00000100:00000001:1.0:1787696015.226659:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298586937 00000100:00000001:1.0:1787696015.226662:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298575555 00000100:00000001:1.0:1787696015.226664:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298586937 00000100:00000001:1.0:1787696015.226667:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696015.226669:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.226671:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787696015.226675:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696015/real 1787696015] req@ffff8b14465af800 x1874531139562496/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787696015.226685:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787696015.226686:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787696015.226688:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787696015.226690:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787696015.226692:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787696015.226694:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787696015.226697:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14465af800 x1874531139562496/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787696015.226707:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14465af800 x1874531139562496/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.226715:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787696015.226720:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.226722:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696015.226724:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787696015.226728:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14465af800 x1874531139562496/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.226735:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787696015.226740:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14465af800 x1874531139562496/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.226747:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787696015.226749:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.226751:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696015.226752:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.226754:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787696015.226756:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787696015.226759:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787696015.226763:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787696015.226765:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787696015.226770:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14465af800 x1874531139562496/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787696015.226779:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14465af800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139562496:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696015.226783:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.226785:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696015.226788:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465ad880 x1874531139562368/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.226795:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787696015.226797:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787696015.226799:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787696015.226802:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787696015.226803:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787696015.226808:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787696015.226811:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787696015.226813:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445662000. 02000000:00000001:1.0:1787696015.226815:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787696015.226817:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787696015.226818:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465ad880. 00000100:00000001:1.0:1787696015.226821:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787696015.226823:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.226825:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696015.226829:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14465af800 x1874531139562496/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696040 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696015.226836:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787696015.226838:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787696015.226839:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546800. 02000000:00000001:1.0:1787696015.226842:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787696015.226843:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787696015.226844:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787696015.226846:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787696015.226848:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1445660c00. 02000000:00000001:1.0:1787696015.226850:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787696015.226851:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787696015.226852:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14465af800. 00000100:00000001:1.0:1787696015.226854:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787696015.226856:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.226858:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696015.226862:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787696015.226864:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696015.226866:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787696015.226868:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787696015.286460:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787696015.286470:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787696015.286472:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787696015.286477:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:0.0:1787696015.286481:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787696015.286485:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696015.286488:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787696015.286490:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696015.286492:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696015.926474:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696015.926483:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696015.926490:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696015.926492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696015.926497:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696015.926498:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696015.926503:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696015.926504:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696015.926530:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696015.926532:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696015.926536:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696015.926537:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787696015.926583:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787696015.926587:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787696015.926591:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787696016.310547:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787696016.310558:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787696016.310561:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787696016.310594:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:0.0:1787696016.310599:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787696016.310605:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696016.310608:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787696016.310612:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696016.310615:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696016.950520:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696016.950530:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696016.950539:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696016.950541:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696016.950546:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696016.950551:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787696016.950554:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787696016.950562:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787696016.950565:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787696016.950566:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696016.950570:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787696016.950572:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696016.950574:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696016.950576:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696016.950579:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696016.950581:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787696016.950582:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787696016.950584:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787696016.950585:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787696016.950585:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696016.950587:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787696016.950588:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696016.950589:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696016.950590:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696016.950593:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696016.950593:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696016.950596:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696016.950597:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787696016.950598:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787696016.950599:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787696016.950600:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787696016.950601:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696016.950602:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787696016.950602:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696016.950603:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696016.950626:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787696016.950699:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787696016.950703:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787696016.950706:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787696017.334473:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787696017.334482:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787696017.334484:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787696017.334490:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:0.0:1787696017.334494:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787696017.334497:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696017.334501:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787696017.334503:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696017.334506:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696017.974484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696017.974490:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696017.974495:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696017.974497:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696017.974499:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696017.974500:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696017.974502:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696017.974503:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696017.974506:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696017.974507:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696017.974510:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696017.974511:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787696017.974584:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787696017.974588:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787696017.974593:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787696018.358481:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787696018.358488:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787696018.358490:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787696018.358495:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:0.0:1787696018.358498:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787696018.358501:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696018.358503:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787696018.358505:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696018.358507:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696018.998523:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696018.998530:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696018.998536:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696018.998537:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696018.998540:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696018.998541:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696018.998544:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696018.998545:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696018.998548:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696018.998549:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696018.998552:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696018.998553:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787696018.998646:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787696018.998655:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787696018.998659:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787696019.382429:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787696019.382436:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787696019.382438:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787696019.382443:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:0.0:1787696019.382446:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787696019.382448:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696019.382450:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787696019.382452:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696019.382470:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696020.022496:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696020.022502:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696020.022506:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696020.022507:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696020.022510:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696020.022511:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696020.022514:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696020.022515:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696020.022517:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696020.022518:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696020.022521:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696020.022521:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787696020.022587:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787696020.022592:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787696020.022594:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787696020.343216:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787696020.343227:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787696020.343231:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787696020.343235:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787696020.343239:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787696020.343246:0:47074:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3908 00000400:00000200:3.0:1787696020.343255:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787696020.343262:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787696020.343265:0:47074:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787696020.343269:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696020.343287:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787696020.343290:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696020.343293:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787696020.343297:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787696020.343299:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:3.0:1787696020.343302:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787696020.343305:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787696020.343308:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787696020.343309:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787696020.343312:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696020.343314:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787696020.343315:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696020.343316:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787696020.343318:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787696020.343321:0:47074:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787696020.343322:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787696020.343325:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787696020.343328:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787696020.343330:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696020.343331:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787696020.343333:0:47074:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787696020.343338:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7480. 00000020:00000040:3.0:1787696020.343340:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787696020.343343:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787696020.343344:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696020.343347:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787696020.343349:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787696020.343353:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787696020.343359:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674800. 00000100:00000001:3.0:1787696020.343396:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787696020.343405:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7480 x1874531139562624/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787696020.343423:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787696020.343426:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787696020.343429:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000043daeda0] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7480 x1874531139562624/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787696020.343448:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787696020.343450:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787696020.343453:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787696020.343454:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787696020.343455:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787696020.343458:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787696020.343459:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3908 00000400:00000200:3.0:1787696020.343463:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787696020.343466:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787696020.343467:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787696020.343470:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696020.343472:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787696020.343474:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696020.343476:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787696020.343477:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787696020.343478:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000100:00000001:1.0:1787696020.343480:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787696020.343480:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787696020.343483:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:3.0:1787696020.343484:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:3.0:1787696020.343485:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:3.0:1787696020.343488:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696020.343490:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787696020.343492:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000040:3.0:1787696020.343493:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696020.343495:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787696020.343496:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000040:1.0:1787696020.343503:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7480 x1874531139562624/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1787696020.343505:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:3.0:1787696020.343512:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787696020.343517:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.343521:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787696020.343521:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787696020.343524:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787696020.343525:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696020.343527:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787696020.343530:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787696020.343531:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696020.343535:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696020.343536:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787696020.343543:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787696020.343544:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139562624:192.168.201.140@tcp:250:kworker.0 00000008:00000001:3.0:1787696020.343548:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.343553:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787696020.343554:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787696020.343560:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4e00. 00000020:00000040:3.0:1787696020.343563:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:3.0:1787696020.343566:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787696020.343568:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696020.343570:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787696020.343570:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787696020.343573:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787696020.343575:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696020.343578:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787696020.343581:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 00000100:00000001:3.0:1787696020.343587:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696020.343589:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1787696020.343594:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4e00 x1874531139562752/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000010:1.0:1787696020.343597:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546800. 02000000:00000001:1.0:1787696020.343602:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696020.343608:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787696020.343610:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696020.343613:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787696020.343617:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fcd77cd6] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4e00 x1874531139562752/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1787696020.343619:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787696020.343627:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000001:3.0:1787696020.343628:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787696020.343632:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787696020.343635:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000200:1.0:1787696020.343636:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139562624, portal 25 00000100:00000001:3.0:1787696020.343637:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787696020.343640:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787696020.343641:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787696020.343645:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930157184 : -128555779394432 : ffff8b14466f7480) 00000100:00000001:3.0:1787696020.343645:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000040:1.0:1787696020.343653:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7480 x1874531139562624/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787696020.343655:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3908 00000400:00000200:3.0:1787696020.343665:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696020.343670:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:3.0:1787696020.343671:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787696020.343675:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:1.0:1787696020.343679:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787696020.343682:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000010:1.0:1787696020.343684:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000001:3.0:1787696020.343685:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787696020.343689:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000200:1.0:1787696020.343690:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139562624, offset 0 00000100:00000001:3.0:1787696020.343692:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787696020.343695:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787696020.343696:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:3.0:1787696020.343697:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787696020.343700:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000400:00000200:1.0:1787696020.343701:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000020:00000001:3.0:1787696020.343706:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:3.0:1787696020.343710:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000400:00000200:1.0:1787696020.343711:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787696020.343718:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:1.0:1787696020.343719:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000100:00000001:3.0:1787696020.343726:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787696020.343728:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000040:3.0:1787696020.343733:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:1.0:1787696020.343734:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787696020.343745:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787696020.343750:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000100:00000001:3.0:1787696020.343752:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000400:00000200:1.0:1787696020.343758:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17205:17205:256:4294967295] 192.168.201.140@tcp LPNI seq info [17204:17204:8:4294967295] 00000020:00000040:3.0:1787696020.343759:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:3.0:1787696020.343765:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787696020.343769:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787696020.343771:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000200:1.0:1787696020.343773:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787696020.343775:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787696020.343778:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696020.343779:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787696020.343781:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787696020.343784:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696020.343788:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696020.343793:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000010:3.0:1787696020.343795:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7100. 00000020:00000040:3.0:1787696020.343800:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000800:00000200:1.0:1787696020.343801:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787696020.343807:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000010:1.0:1787696020.343809:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 02000000:00000001:3.0:1787696020.343816:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1787696020.343817:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787696020.343821:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000200:1.0:1787696020.343824:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 02000000:00000001:3.0:1787696020.343825:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787696020.343829:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.1:1787696020.343835:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 02000000:00000010:3.0:1787696020.343836:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676000. 00000800:00000200:1.1:1787696020.343841:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000100:00000001:3.0:1787696020.343843:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787696020.343851:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696020.343855:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000100:00100000:3.0:1787696020.343860:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7100 x1874531139562880/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787696020.343863:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000001:3.0:1787696020.343872:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000200:1.0:1787696020.343875:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1787696020.343877:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696020.343882:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000100:00000040:3.0:1787696020.343885:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000004e8abd7] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7100 x1874531139562880/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1787696020.343886:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787696020.343892:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787696020.343899:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7480 x1874531139562624/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787696020.343901:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787696020.343907:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787696020.343912:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1787696020.343913:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000040:1.0:1787696020.343917:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7480 x1874531139562624/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696045 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.343923:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.343925:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787696020.343925:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787696020.343930:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787696020.343934:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787696020.343935:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787696020.343938:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3908 00000400:00000200:1.0:1787696020.343946:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787696020.343946:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787696020.343952:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00000001:1.0:1787696020.343953:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298592057 00000100:00080000:3.0:1787696020.343954:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787696020.343960:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696020.343961:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787696020.343963:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696020.343965:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787696020.343966:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298586937 00000020:00000001:3.0:1787696020.343967:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787696020.343969:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:3.0:1787696020.343973:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787696020.343975:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000100:00000001:1.0:1787696020.343976:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298592057 00000020:00000001:3.0:1787696020.343977:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:1.0:1787696020.343983:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298586937 00000020:00000001:3.0:1787696020.343985:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000001:1.0:1787696020.343988:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:3.0:1787696020.343990:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696020.343991:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787696020.343993:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696020.343994:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787696020.343999:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00000001:1.0:1787696020.344000:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787696020.344003:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:1.0:1787696020.344005:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787696020.344006:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787696020.344009:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1787696020.344012:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696020/real 1787696020] req@ffff8b14466f7480 x1874531139562624/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787696020.344013:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787696020.344017:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696020.344021:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787696020.344024:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787696020.344028:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787696020.344029:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787696020.344030:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787696020.344032:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787696020.344035:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000100:00000010:3.0:1787696020.344036:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6a00. 00000400:00000200:1.0:1787696020.344038:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000020:00000040:3.0:1787696020.344039:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 00000400:00000010:1.0:1787696020.344041:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 02000000:00000001:3.0:1787696020.344043:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787696020.344045:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.344046:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787696020.344048:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787696020.344051:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000200:1.0:1787696020.344052:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7480 x1874531139562624/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787696020.344053:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787696020.344060:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 00000100:00000001:3.0:1787696020.344064:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1787696020.344065:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7480 x1874531139562624/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787696020.344089:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6a00 x1874531139563008/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787696020.344094:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787696020.344102:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787696020.344103:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787696020.344105:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696020.344109:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787696020.344109:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696020.344121:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7480 x1874531139562624/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787696020.344121:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002ad47e8f] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6a00 x1874531139563008/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787696020.344140:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.344144:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787696020.344149:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7480 x1874531139562624/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787696020.344156:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787696020.344162:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1787696020.344166:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787696020.344166:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787696020.344167:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787696020.344173:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787696020.344174:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787696020.344178:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3908 00000100:00000001:1.0:1787696020.344180:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696020.344184:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1787696020.344188:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696020.344190:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787696020.344195:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000400:00000200:3.0:1787696020.344197:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:1.0:1787696020.344199:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787696020.344203:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787696020.344206:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000040:3.0:1787696020.344210:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696020.344214:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787696020.344216:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:3.0:1787696020.344219:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000040:1.0:1787696020.344225:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7480 x1874531139562624/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787696020.344226:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787696020.344231:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787696020.344234:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:3.0:1787696020.344236:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787696020.344240:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000100:00100000:1.0:1787696020.344243:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139562624:192.168.201.140@tcp:250:kworker.0 00000020:00000001:3.0:1787696020.344244:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:3.0:1787696020.344248:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000040:3.0:1787696020.344253:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696020.344255:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787696020.344259:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1787696020.344259:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696020.344263:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787696020.344264:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000040:1.0:1787696020.344268:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7480 x1874531139562624/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1787696020.344270:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00000001:1.0:1787696020.344280:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00080000:3.0:1787696020.344280:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 02000000:00000001:1.0:1787696020.344283:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1787696020.344286:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787696020.344290:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000010:1.0:1787696020.344291:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546800. 02000000:00000001:3.0:1787696020.344295:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787696020.344298:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696020.344299:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1787696020.344300:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000020:00000001:1.0:1787696020.344302:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787696020.344304:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:1.0:1787696020.344306:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 00000008:00100000:3.0:1787696020.344310:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 02000000:00000010:1.0:1787696020.344311:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674800. 00000008:00000001:3.0:1787696020.344316:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696020.344317:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1787696020.344318:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696020.344321:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1787696020.344323:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5c00. 00000100:00000010:1.0:1787696020.344324:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7480. 00000020:00000040:3.0:1787696020.344328:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 00000100:00000001:1.0:1787696020.344329:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787696020.344333:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787696020.344333:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787696020.344335:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787696020.344336:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696020.344339:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787696020.344340:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787696020.344342:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.344343:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000010:3.0:1787696020.344345:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 00000100:00000001:1.0:1787696020.344346:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696020.344350:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.344351:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787696020.344354:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787696020.344356:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00100000:3.0:1787696020.344359:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5c00 x1874531139563136/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787696020.344384:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787696020.344386:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787696020.344390:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005bfbcee1] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5c00 x1874531139563136/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787696020.344397:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787696020.344400:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:1.0:1787696020.344401:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4e00 x1874531139562752/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787696020.344407:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787696020.344411:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787696020.344414:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787696020.344417:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787696020.344418:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787696020.344419:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696020.344423:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1787696020.344424:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3908 02000000:00000001:1.0:1787696020.344427:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1787696020.344429:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787696020.344432:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787696020.344435:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1787696020.344438:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787696020.344440:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696020.344443:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1787696020.344446:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139562752:192.168.201.140@tcp:38:kworker.0 00000100:00000040:3.0:1787696020.344448:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696020.344451:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787696020.344452:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787696020.344454:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696020.344459:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787696020.344459:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787696020.344463:0:47074:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000010:1.0:1787696020.344464:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546800. 00000020:00000040:3.0:1787696020.344465:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 02000000:00000001:1.0:1787696020.344469:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787696020.344470:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787696020.344473:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000400:00000200:1.0:1787696020.344478:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000020:00000001:3.0:1787696020.344478:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000020:00000001:3.0:1787696020.344482:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000400:00000010:1.0:1787696020.344483:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000100:00000040:3.0:1787696020.344487:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000010:1.0:1787696020.344490:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000001:3.0:1787696020.344490:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787696020.344495:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000200:1.0:1787696020.344497:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139562752, portal 10 00000100:00000001:3.0:1787696020.344497:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787696020.344502:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000040:3.0:1787696020.344503:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00000001:1.0:1787696020.344506:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930147328 : -128555779404288 : ffff8b14466f4e00) 00000100:00080000:3.0:1787696020.344508:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787696020.344511:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787696020.344513:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000040:1.0:1787696020.344515:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4e00 x1874531139562752/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787696020.344517:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787696020.344521:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696020.344526:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787696020.344530:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787696020.344531:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:3.0:1787696020.344536:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696020.344537:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787696020.344542:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787696020.344543:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787696020.344549:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139562752, offset 0 00000100:00000010:3.0:1787696020.344549:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7800. 00000020:00000040:3.0:1787696020.344555:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000400:00000200:1.0:1787696020.344562:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 02000000:00000001:3.0:1787696020.344563:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787696020.344566:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696020.344569:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000200:1.0:1787696020.344570:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787696020.344571:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787696020.344576:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696020.344580:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 02000000:00000010:3.0:1787696020.344583:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675800. 00000400:00000200:1.0:1787696020.344585:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000100:00000001:3.0:1787696020.344588:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696020.344591:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000100:00100000:3.0:1787696020.344596:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7800 x1874531139563264/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787696020.344597:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787696020.344602:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787696020.344609:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17206:17206:256:4294967295] 192.168.201.140@tcp LPNI seq info [17205:17205:8:4294967295] 00000400:00000200:1.0:1787696020.344613:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696020.344628:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:3.0:1787696020.344631:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000200:1.0:1787696020.344636:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000100:00000001:3.0:1787696020.344636:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787696020.344640:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d7c57981] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7800 x1874531139563264/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000010:1.0:1787696020.344650:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000100:00000001:3.0:1787696020.344650:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787696020.344654:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787696020.344659:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000200:1.0:1787696020.344660:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00000001:3.0:1787696020.344661:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787696020.344665:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.0:1787696020.344666:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000100:00000001:3.0:1787696020.344669:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000800:00000200:1.1:1787696020.344673:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00080000:3.0:1787696020.344675:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3908 00000800:00000200:1.1:1787696020.344679:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000400:00000200:3.0:1787696020.344683:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787696020.344690:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000800:00000100:1.0:1787696020.344691:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000100:00080000:3.0:1787696020.344693:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000800:00000010:1.0:1787696020.344694:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787696020.344700:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000100:00000040:3.0:1787696020.344701:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000400:00000200:1.0:1787696020.344706:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1787696020.344708:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787696020.344711:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000100:00000040:3.0:1787696020.344714:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000010:1.0:1787696020.344715:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787696020.344720:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787696020.344721:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787696020.344726:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000200:1.0:1787696020.344727:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4e00 x1874531139562752/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1787696020.344729:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000020:00000001:3.0:1787696020.344733:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787696020.344737:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:3.0:1787696020.344742:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:1.0:1787696020.344744:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1787696020.344746:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000040:1.0:1787696020.344751:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4e00 x1874531139562752/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696045 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787696020.344753:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696020.344757:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787696020.344760:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696020.344764:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787696020.344769:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1787696020.344772:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00000001:1.0:1787696020.344774:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1787696020.344777:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787696020.344781:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.344782:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787696020.344782:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787696020.344786:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1787696020.344788:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 02000000:00000001:3.0:1787696020.344789:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696020.344794:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:1.0:1787696020.344797:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000008:00100000:3.0:1787696020.344798:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787696020.344802:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298581959 00000008:00000001:3.0:1787696020.344803:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696020.344806:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.344808:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298592057 00000100:00000010:3.0:1787696020.344810:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f7b80. 00000100:00000001:1.0:1787696020.344813:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298581959 00000020:00000040:3.0:1787696020.344813:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 02000000:00000001:3.0:1787696020.344817:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787696020.344820:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.344821:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298592057 00000100:00000001:3.0:1787696020.344822:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787696020.344824:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787696020.344826:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.344828:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000010:3.0:1787696020.344831:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 00000100:00000001:1.0:1787696020.344833:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696020.344835:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.344838:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787696020.344841:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f7b80 x1874531139563392/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787696020.344848:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696020/real 1787696020] req@ffff8b14466f4e00 x1874531139562752/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787696020.344849:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787696020.344852:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787696020.344858:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b87e1da] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f7b80 x1874531139563392/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787696020.344875:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787696020.344878:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000040:3.0:1787696020.344888:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000001:1.0:1787696020.344895:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000040:3.0:1787696020.344895:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:1.0:1787696020.344898:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787696020.344901:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000100:00000040:3.0:1787696020.344902:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1787696020.344905:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787696020.344906:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787696020.344908:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787696020.344913:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4e00 x1874531139562752/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787696020.344917:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3918) 00000100:00000200:1.0:1787696020.344922:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4e00 x1874531139562752/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.344928:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787696020.344929:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.344931:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696020.344932:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787696020.344937:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4e00 x1874531139562752/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.344944:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787696020.344947:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4e00 x1874531139562752/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.344955:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787696020.344957:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.344960:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696020.344961:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.344963:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787696020.344965:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787696020.344968:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787696020.344971:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787696020.344974:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787696020.344985:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4e00 x1874531139562752/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787696020.344991:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139562752:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696020.344994:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787696020.344996:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7100 x1874531139562880/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345000:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787696020.345001:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696020.345003:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787696020.345004:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696020.345006:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696020.345008:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139562880:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696020.345010:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787696020.345011:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787696020.345013:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696020.345014:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787696020.345016:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787696020.345018:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696020.345021:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787696020.345023:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787696020.345027:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787696020.345030:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139562880, portal 10 00000100:00000001:1.0:1787696020.345032:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787696020.345033:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930156288 : -128555779395328 : ffff8b14466f7100) 00000100:00000040:1.0:1787696020.345037:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7100 x1874531139562880/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345042:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787696020.345043:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787696020.345045:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787696020.345047:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139562880, offset 0 00000400:00000200:1.0:1787696020.345053:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787696020.345057:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696020.345059:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787696020.345061:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787696020.345063:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787696020.345065:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787696020.345067:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787696020.345069:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17207:17207:256:4294967295] 192.168.201.140@tcp LPNI seq info [17206:17206:8:4294967295] 00000400:00000200:1.0:1787696020.345085:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696020.345090:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787696020.345094:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696020.345096:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787696020.345099:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787696020.345101:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1787696020.345104:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787696020.345106:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000800:00000100:1.0:1787696020.345109:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696020.345110:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787696020.345113:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787696020.345115:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787696020.345117:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787696020.345119:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787696020.345121:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787696020.345123:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7100 x1874531139562880/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345129:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696020.345131:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7100 x1874531139562880/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696045 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345136:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.345138:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787696020.345141:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696020.345144:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787696020.345147:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696020.345155:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298580807 00000100:00000001:1.0:1787696020.345158:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298592057 00000100:00000001:1.0:1787696020.345160:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298580807 00000100:00000001:1.0:1787696020.345162:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298592057 00000100:00000001:1.0:1787696020.345165:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696020.345167:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.345169:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787696020.345174:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696020/real 1787696020] req@ffff8b14466f7100 x1874531139562880/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787696020.345183:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787696020.345185:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787696020.345186:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787696020.345189:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787696020.345191:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787696020.345193:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787696020.345195:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7100 x1874531139562880/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787696020.345201:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7100 x1874531139562880/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345205:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787696020.345206:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.345207:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696020.345208:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787696020.345210:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7100 x1874531139562880/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345215:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787696020.345217:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7100 x1874531139562880/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345222:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787696020.345227:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.345228:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696020.345229:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.345230:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787696020.345232:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787696020.345233:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787696020.345236:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787696020.345237:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787696020.345240:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7100 x1874531139562880/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787696020.345245:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139562880:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696020.345248:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787696020.345250:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6a00 x1874531139563008/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345254:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787696020.345255:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696020.345256:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787696020.345258:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696020.345259:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696020.345261:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139563008:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787696020.345263:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787696020.345264:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787696020.345265:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696020.345266:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787696020.345268:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547400. 02000000:00000001:1.0:1787696020.345270:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696020.345273:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787696020.345274:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787696020.345277:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787696020.345279:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139563008, portal 4 00000100:00000001:1.0:1787696020.345283:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787696020.345284:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930154496 : -128555779397120 : ffff8b14466f6a00) 00000100:00000040:1.0:1787696020.345289:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6a00 x1874531139563008/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345293:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787696020.345294:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787696020.345296:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787696020.345298:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139563008, offset 0 00000400:00000200:1.0:1787696020.345301:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787696020.345303:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696020.345306:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787696020.345308:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787696020.345310:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787696020.345311:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787696020.345313:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787696020.345315:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17208:17208:256:4294967295] 192.168.201.140@tcp LPNI seq info [17207:17207:8:4294967295] 00000400:00000200:1.0:1787696020.345318:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696020.345322:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787696020.345325:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696020.345336:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787696020.345342:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787696020.345344:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1787696020.345347:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787696020.345349:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000800:00000100:1.0:1787696020.345351:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696020.345353:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787696020.345355:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787696020.345357:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787696020.345360:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787696020.345377:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787696020.345379:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787696020.345382:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6a00 x1874531139563008/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345398:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696020.345400:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6a00 x1874531139563008/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696045 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345405:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.345406:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787696020.345410:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696020.345412:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787696020.345415:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696020.345417:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298580838 00000100:00000001:1.0:1787696020.345419:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298592057 00000100:00000001:1.0:1787696020.345421:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298580838 00000100:00000001:1.0:1787696020.345423:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298592057 00000100:00000001:1.0:1787696020.345424:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696020.345426:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.345427:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787696020.345430:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696020/real 1787696020] req@ffff8b14466f6a00 x1874531139563008/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787696020.345435:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787696020.345435:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787696020.345436:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787696020.345438:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787696020.345439:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787696020.345441:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787696020.345443:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6a00 x1874531139563008/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787696020.345448:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6a00 x1874531139563008/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345452:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787696020.345457:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.345458:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696020.345459:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787696020.345462:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6a00 x1874531139563008/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345467:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787696020.345469:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6a00 x1874531139563008/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345474:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787696020.345475:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.345476:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696020.345477:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.345478:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787696020.345479:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787696020.345481:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787696020.345482:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787696020.345484:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787696020.345486:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6a00 x1874531139563008/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787696020.345491:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139563008:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787696020.345493:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.345494:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696020.345496:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4e00 x1874531139562752/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345500:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787696020.345501:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787696020.345503:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546800. 02000000:00000001:1.0:1787696020.345505:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787696020.345506:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787696020.345509:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787696020.345511:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787696020.345512:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:1.0:1787696020.345516:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787696020.345517:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787696020.345518:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4e00. 00000100:00000001:1.0:1787696020.345520:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787696020.345521:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.345522:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696020.345524:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7100 x1874531139562880/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345529:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787696020.345531:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787696020.345532:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787696020.345536:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787696020.345537:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787696020.345538:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787696020.345540:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787696020.345542:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676000. 02000000:00000001:1.0:1787696020.345545:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787696020.345546:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787696020.345547:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7100. 00000100:00000001:1.0:1787696020.345549:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787696020.345551:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.345553:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696020.345556:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6a00 x1874531139563008/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345563:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787696020.345564:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787696020.345566:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547400. 02000000:00000001:1.0:1787696020.345569:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787696020.345570:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787696020.345572:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787696020.345574:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787696020.345575:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:1.0:1787696020.345578:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787696020.345584:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787696020.345585:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6a00. 00000100:00000001:1.0:1787696020.345588:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787696020.345589:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.345590:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.345593:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787696020.345594:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.345595:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787696020.345596:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787696020.345598:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787696020.345600:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5c00 x1874531139563136/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345605:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787696020.345606:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696020.345608:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787696020.345609:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696020.345610:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696020.345612:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139563136:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787696020.345614:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787696020.345616:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787696020.345617:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696020.345618:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787696020.345620:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547400. 02000000:00000001:1.0:1787696020.345621:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696020.345624:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787696020.345625:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787696020.345628:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787696020.345630:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139563136, portal 4 00000100:00000001:1.0:1787696020.345631:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787696020.345632:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150912 : -128555779400704 : ffff8b14466f5c00) 00000100:00000040:1.0:1787696020.345636:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5c00 x1874531139563136/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345640:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787696020.345642:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787696020.345646:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787696020.345648:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139563136, offset 0 00000400:00000200:1.0:1787696020.345651:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787696020.345654:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696020.345656:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787696020.345659:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787696020.345660:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787696020.345662:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787696020.345664:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787696020.345666:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17209:17209:256:4294967295] 192.168.201.140@tcp LPNI seq info [17208:17208:8:4294967295] 00000400:00000200:1.0:1787696020.345668:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696020.345673:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787696020.345676:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696020.345678:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787696020.345680:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787696020.345682:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1787696020.345685:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787696020.345687:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000800:00000100:1.0:1787696020.345690:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696020.345691:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787696020.345693:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787696020.345695:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787696020.345697:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787696020.345698:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787696020.345700:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787696020.345702:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5c00 x1874531139563136/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345708:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696020.345710:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5c00 x1874531139563136/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696045 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345714:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.345715:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787696020.345721:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696020.345723:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787696020.345725:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696020.345726:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298580510 00000100:00000001:1.0:1787696020.345728:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298592057 00000100:00000001:1.0:1787696020.345730:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298580510 00000100:00000001:1.0:1787696020.345732:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298592057 00000100:00000001:1.0:1787696020.345734:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696020.345737:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.345739:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787696020.345743:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696020/real 1787696020] req@ffff8b14466f5c00 x1874531139563136/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787696020.345751:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787696020.345753:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787696020.345754:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787696020.345757:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787696020.345758:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787696020.345760:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787696020.345764:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5c00 x1874531139563136/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787696020.345773:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5c00 x1874531139563136/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345781:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787696020.345782:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.345784:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696020.345786:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787696020.345789:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5c00 x1874531139563136/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345794:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787696020.345797:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5c00 x1874531139563136/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345806:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787696020.345808:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.345809:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696020.345810:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.345811:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787696020.345813:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787696020.345815:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787696020.345818:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787696020.345820:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787696020.345825:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5c00 x1874531139563136/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787696020.345833:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139563136:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787696020.345837:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787696020.345840:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7800 x1874531139563264/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345847:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787696020.345848:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696020.345850:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787696020.345852:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696020.345853:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696020.345856:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139563264:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696020.345859:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787696020.345861:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787696020.345862:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696020.345864:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787696020.345867:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787696020.345869:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696020.345873:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787696020.345880:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787696020.345883:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787696020.345887:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139563264, portal 10 00000100:00000001:1.0:1787696020.345888:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787696020.345890:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930158080 : -128555779393536 : ffff8b14466f7800) 00000100:00000040:1.0:1787696020.345896:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7800 x1874531139563264/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345903:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787696020.345905:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787696020.345908:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787696020.345911:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139563264, offset 0 00000400:00000200:1.0:1787696020.345914:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787696020.345916:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696020.345919:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787696020.345921:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787696020.345923:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787696020.345924:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787696020.345926:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787696020.345928:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17210:17210:256:4294967295] 192.168.201.140@tcp LPNI seq info [17209:17209:8:4294967295] 00000400:00000200:1.0:1787696020.345932:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696020.345939:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787696020.345944:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696020.345948:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787696020.345952:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787696020.345955:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1787696020.345959:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787696020.345962:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000800:00000100:1.0:1787696020.345966:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696020.345968:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787696020.345971:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787696020.345975:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787696020.345978:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787696020.345980:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787696020.345989:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787696020.345992:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7800 x1874531139563264/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.345997:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696020.346000:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7800 x1874531139563264/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696045 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.346004:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.346005:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787696020.346008:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696020.346010:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787696020.346012:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696020.346014:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298580696 00000100:00000001:1.0:1787696020.346015:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298592057 00000100:00000001:1.0:1787696020.346017:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298580696 00000100:00000001:1.0:1787696020.346018:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298592057 00000100:00000001:1.0:1787696020.346020:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696020.346021:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.346022:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787696020.346025:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696020/real 1787696020] req@ffff8b14466f7800 x1874531139563264/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787696020.346029:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787696020.346030:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787696020.346031:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787696020.346033:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787696020.346034:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787696020.346035:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787696020.346037:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7800 x1874531139563264/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787696020.346042:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7800 x1874531139563264/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.346053:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787696020.346054:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.346055:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696020.346056:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787696020.346059:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7800 x1874531139563264/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.346063:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787696020.346066:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7800 x1874531139563264/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.346070:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787696020.346085:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.346086:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696020.346087:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.346088:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787696020.346090:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787696020.346091:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787696020.346094:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787696020.346095:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787696020.346098:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7800 x1874531139563264/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787696020.346103:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139563264:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696020.346106:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787696020.346108:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f7b80 x1874531139563392/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.346112:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787696020.346113:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696020.346118:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787696020.346119:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696020.346121:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696020.346123:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139563392:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696020.346125:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787696020.346127:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787696020.346128:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696020.346129:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787696020.346131:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546800. 02000000:00000001:1.0:1787696020.346132:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696020.346136:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787696020.346138:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787696020.346142:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787696020.346145:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139563392, portal 10 00000100:00000001:1.0:1787696020.346147:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787696020.346148:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930158976 : -128555779392640 : ffff8b14466f7b80) 00000100:00000040:1.0:1787696020.346154:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f7b80 x1874531139563392/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.346160:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787696020.346162:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787696020.346165:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787696020.346167:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139563392, offset 0 00000400:00000200:1.0:1787696020.346172:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787696020.346176:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696020.346180:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787696020.346183:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787696020.346186:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787696020.346189:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787696020.346191:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787696020.346194:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17211:17211:256:4294967295] 192.168.201.140@tcp LPNI seq info [17210:17210:8:4294967295] 00000400:00000200:1.0:1787696020.346198:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696020.346203:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787696020.346211:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696020.346213:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787696020.346217:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787696020.346220:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000800:00000200:1.1:1787696020.346224:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787696020.346231:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 47 secs later) 00000800:00000100:1.0:1787696020.346236:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696020.346238:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787696020.346241:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787696020.346244:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787696020.346248:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787696020.346250:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787696020.346253:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787696020.346257:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f7b80 x1874531139563392/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696045 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.346265:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696020.346267:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f7b80 x1874531139563392/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696045 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.346274:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.346276:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787696020.346279:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696020.346283:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787696020.346286:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696020.346288:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298580674 00000100:00000001:1.0:1787696020.346291:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298592057 00000100:00000001:1.0:1787696020.346292:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298580674 00000100:00000001:1.0:1787696020.346294:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298592057 00000100:00000001:1.0:1787696020.346296:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696020.346298:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.346300:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787696020.346304:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696020/real 1787696020] req@ffff8b14466f7b80 x1874531139563392/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787696020.346319:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787696020.346320:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787696020.346322:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787696020.346325:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787696020.346327:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787696020.346329:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787696020.346334:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f7b80 x1874531139563392/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787696020.346343:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f7b80 x1874531139563392/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.346350:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787696020.346352:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.346353:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696020.346355:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787696020.346358:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f7b80 x1874531139563392/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.346385:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787696020.346391:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f7b80 x1874531139563392/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.346399:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787696020.346402:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.346403:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696020.346405:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.346407:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787696020.346408:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787696020.346411:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787696020.346413:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787696020.346415:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787696020.346423:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f7b80 x1874531139563392/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787696020.346428:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139563392:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696020.346431:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.346432:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696020.346434:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5c00 x1874531139563136/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.346438:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787696020.346439:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787696020.346441:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547400. 02000000:00000001:1.0:1787696020.346443:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787696020.346444:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787696020.346445:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787696020.346446:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787696020.346448:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:1.0:1787696020.346449:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787696020.346450:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787696020.346451:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5c00. 00000100:00000001:1.0:1787696020.346453:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787696020.346454:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.346455:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696020.346457:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7800 x1874531139563264/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.346460:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787696020.346461:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787696020.346462:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787696020.346464:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787696020.346465:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787696020.346466:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787696020.346467:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787696020.346468:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675800. 02000000:00000001:1.0:1787696020.346470:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787696020.346474:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787696020.346475:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7800. 00000100:00000001:1.0:1787696020.346477:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787696020.346478:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.346479:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696020.346481:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f7b80 x1874531139563392/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696045 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696020.346485:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787696020.346486:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787696020.346487:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546800. 02000000:00000001:1.0:1787696020.346488:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787696020.346489:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787696020.346490:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787696020.346491:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787696020.346491:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:1.0:1787696020.346493:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787696020.346494:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787696020.346494:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f7b80. 00000100:00000001:1.0:1787696020.346496:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787696020.346496:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.346497:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696020.346500:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787696020.346501:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696020.346502:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787696020.346503:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787696020.406482:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787696020.406488:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787696020.406490:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787696020.406495:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:0.0:1787696020.406499:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787696020.406502:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696020.406505:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787696020.406508:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696020.406511:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696021.046478:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696021.046484:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696021.046488:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696021.046489:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696021.046492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696021.046493:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696021.046495:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696021.046496:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696021.046498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696021.046499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696021.046502:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696021.046503:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787696021.046594:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787696021.046597:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787696021.046599:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787696021.430445:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787696021.430452:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787696021.430453:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787696021.430457:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:0.0:1787696021.430460:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787696021.430463:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696021.430465:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787696021.430467:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696021.430469:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696022.070481:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696022.070486:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696022.070491:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696022.070492:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696022.070495:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696022.070496:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696022.070498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696022.070499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696022.070502:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696022.070504:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787696022.070506:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787696022.070510:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787696022.070516:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787696022.070517:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696022.070520:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787696022.070521:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696022.070523:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696022.070524:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696022.070527:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696022.070528:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787696022.070589:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787696022.070593:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787696022.070597:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787696022.454474:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787696022.454481:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787696022.454483:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787696022.454487:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:0.0:1787696022.454490:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787696022.454493:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696022.454495:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787696022.454496:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696022.454499:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696023.094474:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696023.094480:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696023.094484:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696023.094485:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696023.094488:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696023.094490:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696023.094492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696023.094493:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696023.094496:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696023.094497:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696023.094499:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696023.094500:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787696023.094551:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787696023.094555:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787696023.094560:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787696023.478424:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787696023.478429:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787696023.478431:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787696023.478435:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:0.0:1787696023.478452:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787696023.478455:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696023.478457:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787696023.478459:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696023.478461:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696024.118571:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696024.118580:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696024.118586:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696024.118587:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696024.118590:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696024.118591:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696024.118618:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696024.118620:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696024.118623:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696024.118624:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696024.118627:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696024.118628:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787696024.118717:0:12596:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787696024.118722:0:12596:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787696024.118726:0:12596:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787696024.502479:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787696024.502487:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787696024.502490:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787696024.502496:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:0.0:1787696024.502500:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787696024.502505:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696024.502507:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787696024.502510:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696024.502513:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696025.142455:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696025.142462:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787696025.142465:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787696025.142467:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696025.142471:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787696025.142473:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696025.142475:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696025.142477:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696025.142479:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696025.142480:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787696025.142481:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787696025.142482:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696025.142484:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787696025.142484:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696025.142486:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696025.142487:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696025.142489:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696025.142490:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696025.142492:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696025.142493:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696025.142495:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696025.142496:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696025.142498:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696025.142499:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787696025.142545:0:12595:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787696025.142548:0:12595:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787696025.142551:0:12595:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787696025.462491:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7a438299-70c4-4662-a164-008749121369->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787696025.462502:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787696025.462505:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787696025.462509:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787696025.462514:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787696025.462522:0:47074:0:(import.c:510:import_select_connection()) MGC192.168.201.140@tcp: connect to NID 192.168.201.140@tcp last attempt 3913 00000400:00000200:3.0:1787696025.462531:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787696025.462538:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787696025.462541:0:47074:0:(import.c:589:import_select_connection()) MGC192.168.201.140@tcp: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787696025.462545:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.462548:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787696025.462551:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.462553:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787696025.462557:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787696025.462559:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beebed 00000020:00000001:3.0:1787696025.462562:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787696025.462566:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1446a18800 refcount=3 00000020:00000001:3.0:1787696025.462569:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000020:00000001:3.0:1787696025.462571:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615517933438976 : -128555776112640 : ffff8b1446a18800) 00000100:00000040:3.0:1787696025.462575:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.462577:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787696025.462579:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.462581:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787696025.462584:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1446a18800 : new refcount 2 00000100:00080000:3.0:1787696025.462588:0:47074:0:(import.c:624:import_select_connection()) MGC192.168.201.140@tcp: import ffff8b1442fe5000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787696025.462591:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787696025.462602:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787696025.462606:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787696025.462610:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696025.462612:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787696025.462616:0:47074:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787696025.462622:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4a80. 00000020:00000040:3.0:1787696025.462626:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe5000 refcount=4 obd=MGC192.168.201.140@tcp 02000000:00000001:3.0:1787696025.462630:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787696025.462632:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696025.462635:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787696025.462637:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787696025.462642:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787696025.462649:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675400. 00000100:00000001:3.0:1787696025.462657:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787696025.462667:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4a80 x1874531139563520/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787696025.462678:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787696025.462683:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787696025.462688:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000034ad0d67] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4a80 x1874531139563520/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787696025.462712:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787696025.462715:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787696025.462717:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787696025.462718:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787696025.462724:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787696025.462727:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787696025.462730:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3913 00000400:00000200:3.0:1787696025.462734:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787696025.462741:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787696025.462743:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787696025.462747:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.462749:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787696025.462759:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.462761:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787696025.462764:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787696025.462765:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec41 00000020:00000001:3.0:1787696025.462767:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787696025.462769:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458085000 refcount=5 00000020:00000001:3.0:1787696025.462772:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000020:00000001:3.0:1787696025.462774:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225387520 : -128555484164096 : ffff8b1458085000) 00000100:00000040:3.0:1787696025.462777:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.462779:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787696025.462780:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.462782:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787696025.462785:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458085000 : new refcount 4 00000100:00080000:3.0:1787696025.462788:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b1458183800: import ffff8b1442fe1000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787696025.462790:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787696025.462793:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787696025.462798:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787696025.462800:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696025.462803:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787696025.462806:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787696025.462810:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696025.462811:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787696025.462814:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f6680. 00000020:00000040:3.0:1787696025.462816:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1442fe1000 refcount=8 obd=lustre-MDT0000-mdc-ffff8b1458183800 02000000:00000001:3.0:1787696025.462818:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787696025.462819:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696025.462820:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787696025.462821:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787696025.462823:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787696025.462828:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677400. 00000100:00000001:3.0:1787696025.462830:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787696025.462834:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f6680 x1874531139563648/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787696025.462844:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787696025.462846:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787696025.462849:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f6d2abf9] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f6680 x1874531139563648/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787696025.462854:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787696025.462856:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787696025.462859:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787696025.462860:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787696025.462861:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787696025.462863:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787696025.462865:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3913 00000400:00000200:3.0:1787696025.462868:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787696025.462870:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787696025.462871:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787696025.462873:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.462874:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787696025.462875:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.462877:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787696025.462879:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787696025.462880:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec48 00000020:00000001:3.0:1787696025.462881:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787696025.462882:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458081000 refcount=5 00000020:00000001:3.0:1787696025.462883:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000020:00000001:3.0:1787696025.462885:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225371136 : -128555484180480 : ffff8b1458081000) 00000100:00000040:3.0:1787696025.462886:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.462887:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787696025.462889:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.462890:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787696025.462895:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458081000 : new refcount 4 00000100:00080000:3.0:1787696025.462897:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b1458183800: import ffff8b1458081800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787696025.462899:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787696025.462900:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787696025.462902:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787696025.462903:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696025.462905:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787696025.462906:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787696025.462909:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696025.462911:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787696025.462914:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5880. 00000020:00000040:3.0:1787696025.462917:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458081800 refcount=8 obd=lustre-MDT0001-mdc-ffff8b1458183800 02000000:00000001:3.0:1787696025.462919:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787696025.462921:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696025.462922:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787696025.462924:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787696025.462925:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787696025.462929:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446675800. 00000100:00000001:3.0:1787696025.462933:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787696025.462938:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5880 x1874531139563776/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787696025.462946:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787696025.462949:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787696025.462953:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000827e7dbe] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5880 x1874531139563776/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787696025.462962:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787696025.462965:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787696025.462967:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787696025.462969:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787696025.462970:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787696025.462972:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787696025.462978:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3913 00000400:00000200:3.0:1787696025.462981:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787696025.462999:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787696025.463002:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787696025.463005:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.463007:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787696025.463010:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.463012:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787696025.463014:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787696025.463020:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec56 00000020:00000001:3.0:1787696025.463022:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787696025.463024:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1458082800 refcount=4 00000020:00000001:3.0:1787696025.463026:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000020:00000001:3.0:1787696025.463028:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518225377280 : -128555484174336 : ffff8b1458082800) 00000100:00000040:3.0:1787696025.463031:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.463033:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787696025.463035:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.463037:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787696025.463039:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1458082800 : new refcount 3 00000100:00080000:3.0:1787696025.463042:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8b1458183800: import ffff8b1458083000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787696025.463045:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787696025.463047:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787696025.463050:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787696025.463052:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696025.463054:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787696025.463056:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787696025.463058:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696025.463060:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787696025.463063:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4000. 00000020:00000040:3.0:1787696025.463066:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458083000 refcount=3 obd=lustre-OST0000-osc-ffff8b1458183800 02000000:00000001:3.0:1787696025.463069:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787696025.463080:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696025.463083:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787696025.463085:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787696025.463087:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1787696025.463091:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676400. 00000100:00000001:3.0:1787696025.463094:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787696025.463100:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4000 x1874531139563904/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787696025.463107:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787696025.463110:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787696025.463113:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000b5a8f356] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4000 x1874531139563904/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787696025.463121:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787696025.463123:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 33b90c3f-84c2-4e9c-a496-7dca4143e2a1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787696025.463126:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1787696025.463128:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787696025.463130:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787696025.463132:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787696025.463134:0:47074:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: connect to NID 192.168.201.140@tcp last attempt 3913 00000400:00000200:3.0:1787696025.463139:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787696025.463143:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787696025.463145:0:47074:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787696025.463152:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.463158:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787696025.463160:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.463162:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787696025.463165:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787696025.463166:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3beec5d 00000020:00000001:3.0:1787696025.463168:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000040:3.0:1787696025.463170:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b1453377800 refcount=4 00000020:00000001:3.0:1787696025.463172:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000020:00000001:3.0:1787696025.463182:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518144591872 : -128555564959744 : ffff8b1453377800) 00000100:00000001:1.0:1787696025.463185:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1787696025.463185:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.463187:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787696025.463189:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:1.0:1787696025.463190:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787696025.463191:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000100:00000001:1.0:1787696025.463192:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000040:3.0:1787696025.463194:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b1453377800 : new refcount 3 00000100:00080000:3.0:1787696025.463196:0:47074:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff8b1458183800: import ffff8b1458080800 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000040:1.0:1787696025.463199:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4a80 x1874531139563520/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787696025.463199:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787696025.463201:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787696025.463204:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787696025.463206:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696025.463209:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787696025.463209:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787696025.463211:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787696025.463211:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683216 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787696025.463214:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696025.463216:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787696025.463219:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f4700. 02000000:00000001:1.0:1787696025.463220:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:3.0:1787696025.463221:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b1458080800 refcount=3 obd=lustre-OST0001-osc-ffff8b1458183800 02000000:00000001:1.0:1787696025.463223:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787696025.463224:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787696025.463225:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787696025.463226:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696025.463228:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:1.0:1787696025.463229:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139563520:192.168.201.140@tcp:250:kworker.0 02000000:00000001:3.0:1787696025.463229:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787696025.463231:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696025.463234:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000010:3.0:1787696025.463235:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446674000. 00000100:00000001:3.0:1787696025.463238:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696025.463239:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787696025.463241:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787696025.463243:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f4700 x1874531139564032/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787696025.463244:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787696025.463248:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446546800. 00000100:00000001:3.0:1787696025.463250:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787696025.463251:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696025.463252:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696025.463256:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000040:3.0:1787696025.463256:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008818c2cc] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f4700 x1874531139564032/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1787696025.463261:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000100:00000001:3.0:1787696025.463265:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1787696025.463268:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00080000:3.0:1787696025.463271:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787696025.463274:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000200:1.0:1787696025.463275:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139563520, portal 25 00000100:00000001:3.0:1787696025.463276:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787696025.463278:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787696025.463278:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787696025.463280:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930146432 : -128555779405184 : ffff8b14466f4a80) 00000100:00000001:3.0:1787696025.463281:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787696025.463283:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3913 00000100:00000040:1.0:1787696025.463285:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4a80 x1874531139563520/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696050 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1787696025.463287:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:3.0:1787696025.463290:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787696025.463293:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000100:00000040:3.0:1787696025.463296:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.463298:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787696025.463300:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.463303:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787696025.463305:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787696025.463306:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000040:3.0:1787696025.463306:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa670 00000020:00000001:3.0:1787696025.463308:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000040:1.0:1787696025.463309:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000020:00000040:3.0:1787696025.463311:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606bf800 refcount=5 00000400:00000010:1.0:1787696025.463312:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000020:00000001:3.0:1787696025.463313:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000020:00000001:3.0:1787696025.463315:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366136320 : -128555343415296 : ffff8b14606bf800) 00000100:00000200:1.0:1787696025.463316:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1874531139563520, offset 0 00000100:00000040:3.0:1787696025.463318:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.463320:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000200:1.0:1787696025.463322:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000100:00000040:3.0:1787696025.463322:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.463324:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787696025.463326:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606bf800 : new refcount 4 00000100:00080000:3.0:1787696025.463329:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8b145807d000: import ffff8b144515e000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000400:00000200:1.0:1787696025.463330:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:3.0:1787696025.463331:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787696025.463333:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787696025.463335:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1787696025.463336:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 02000000:00000001:3.0:1787696025.463337:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696025.463340:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000200:1.0:1787696025.463342:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000008:00100000:3.0:1787696025.463345:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000400:00000200:1.0:1787696025.463347:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000008:00000001:3.0:1787696025.463348:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696025.463349:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696025.463351:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000100:00000010:3.0:1787696025.463352:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5500. 00000400:00000200:1.0:1787696025.463354:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000020:00000040:3.0:1787696025.463355:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515e000 refcount=6 obd=lustre-MDT0000-mdc-ffff8b145807d000 02000000:00000001:3.0:1787696025.463357:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000200:1.0:1787696025.463358:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17212:17212:256:4294967295] 192.168.201.140@tcp LPNI seq info [17211:17211:8:4294967295] 02000000:00000001:3.0:1787696025.463359:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696025.463361:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787696025.463385:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787696025.463388:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696025.463392:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000010:3.0:1787696025.463393:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446676000. 00000100:00000001:3.0:1787696025.463396:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787696025.463402:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5500 x1874531139564160/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000200:1.0:1787696025.463404:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000100:00000001:3.0:1787696025.463410:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787696025.463413:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787696025.463417:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000002a777d68] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5500 x1874531139564160/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1787696025.463418:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696025.463424:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000100:00000001:3.0:1787696025.463425:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787696025.463428:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000200:1.0:1787696025.463431:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000100:00080000:3.0:1787696025.463431:0:47074:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787696025.463433:0:47074:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787696025.463434:0:47074:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000200:1.0:1787696025.463435:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000100:00000001:3.0:1787696025.463438:0:47074:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787696025.463440:0:47074:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: connect to NID 192.168.201.140@tcp last attempt 3913 00000800:00000200:1.1:1787696025.463441:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000400:00000200:3.0:1787696025.463444:0:47074:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000800:00000200:1.1:1787696025.463445:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000400:00000200:3.0:1787696025.463447:0:47074:0:(peer.c:1685:LNetPeerDiscovered()) Peer NID 192.168.201.140@tcp is discovered: rc = 1 00000100:00080000:3.0:1787696025.463449:0:47074:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787696025.463450:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696025.463453:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000100:00000040:3.0:1787696025.463453:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.463455:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787696025.463457:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000400:00000200:1.0:1787696025.463458:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787696025.463463:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000100:00000001:3.0:1787696025.463463:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000001:3.0:1787696025.463466:0:47074:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000040:3.0:1787696025.463467:0:47074:0:(genops.c:804:class_conn2export()) looking for export cookie 0xae91b608b3bfa677 00000400:00000200:1.0:1787696025.463469:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000020:00000001:3.0:1787696025.463469:0:47074:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000010:1.0:1787696025.463471:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000020:00000040:3.0:1787696025.463471:0:47074:0:(lustre_handles.c:148:class_handle2object()) GET export ffff8b14606be000 refcount=5 00000020:00000001:3.0:1787696025.463473:0:47074:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000020:00000001:3.0:1787696025.463475:0:47074:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446615518366130176 : -128555343421440 : ffff8b14606be000) 00000100:00000001:1.0:1787696025.463476:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000040:3.0:1787696025.463478:0:47074:0:(lustre_net.h:1965:ptlrpc_connection_put()) PUT conn=ffff8b1444613dc0 refcount 26 to 192.168.201.140@tcp 00000100:00000200:1.0:1787696025.463480:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4a80 x1874531139563520/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696050 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787696025.463480:0:47074:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1787696025.463482:0:47074:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff8b1444613dc0 refcount 27 to 192.168.201.140@tcp 00000100:00000001:3.0:1787696025.463485:0:47074:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446615517895671232 : -128555813880384 : ffff8b1444613dc0) 00000020:00000040:3.0:1787696025.463487:0:47074:0:(genops.c:879:class_export_put()) PUTting export ffff8b14606be000 : new refcount 4 00000100:00000001:1.0:1787696025.463490:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1787696025.463490:0:47074:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8b145807d000: import ffff8b144515c000 using connection 192.168.201.140@tcp/192.168.201.140@tcp 00000100:00000001:3.0:1787696025.463492:0:47074:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696025.463494:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4a80 x1874531139563520/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 0 dl 1787696050 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787696025.463494:0:47074:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787696025.463497:0:47074:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787696025.463499:0:47074:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696025.463501:0:47074:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787696025.463502:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696025.463504:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000008:00100000:3.0:1787696025.463504:0:47074:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683216 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787696025.463506:0:47074:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696025.463508:0:47074:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696025.463510:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1787696025.463511:0:47074:0:(client.c:606:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff8b14466f5f80. 00000020:00000040:3.0:1787696025.463513:0:47074:0:(genops.c:1210:class_import_get()) import ffff8b144515c000 refcount=9 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000400:00000200:1.0:1787696025.463514:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 02000000:00000001:3.0:1787696025.463515:0:47074:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787696025.463517:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 2 is 4298597177 02000000:00000001:3.0:1787696025.463517:0:47074:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787696025.463519:0:47074:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787696025.463520:0:47074:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787696025.463522:0:47074:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696025.463524:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 2, simply extend its deadline from 4298592057 02000000:00000010:3.0:1787696025.463525:0:47074:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff8b1446677c00. 00000100:00000001:1.0:1787696025.463529:0:2402:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS new deadline estimation for cpu 3 is 4298597177 00000100:00000001:3.0:1787696025.463529:0:47074:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696025.463532:0:2402:0:(niobuf.c:685:kick_cpu_latency()) MGC192.168.201.140@tcp: PM QoS request active for cpu 3, simply extend its deadline from 4298592057 00000100:00100000:3.0:1787696025.463533:0:47074:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8b14466f5f80 x1874531139564288/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787696025.463536:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696025.463539:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696025.463541:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787696025.463546:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696025/real 1787696025] req@ffff8b14466f4a80 x1874531139563520/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787696025.463552:0:47074:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787696025.463554:0:47074:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1787696025.463558:0:47074:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006e7bde76] to pc [ptlrpcd_rcv+-1] req@ffff8b14466f5f80 x1874531139564288/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787696025.463567:0:47074:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787696025.463568:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787696025.463570:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000040:3.0:1787696025.463570:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000010:1.0:1787696025.463572:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000100:00000040:3.0:1787696025.463573:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000400:00000200:1.0:1787696025.463574:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787696025.463577:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000040:3.0:1787696025.463577:0:47074:0:(pinger.c:229:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: level IDLE/11 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787696025.463579:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000040:3.0:1787696025.463580:0:47074:0:(pinger.c:247:ptlrpc_pinger_process_import()) 608c526c-eb6e-4a54-80a5-8969828241eb->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: IDLE) 00000100:00000200:1.0:1787696025.463583:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4a80 x1874531139563520/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1787696025.463585:0:47074:0:(pinger.c:304:ptlrpc_pinger_main()) next wakeup in 5 (3923) 00000100:00000200:1.0:1787696025.463592:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4a80 x1874531139563520/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.463598:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787696025.463600:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.463603:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696025.463605:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787696025.463609:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4a80 x1874531139563520/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.463616:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787696025.463621:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4a80 x1874531139563520/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.463627:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787696025.463631:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.463632:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696025.463634:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.463636:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787696025.463638:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787696025.463641:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe5000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787696025.463645:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787696025.463648:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787696025.463652:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4a80 x1874531139563520/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787696025.463668:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7a438299-70c4-4662-a164-008749121369:2402:1874531139563520:192.168.201.140@tcp:250:kworker.0 00000100:00000001:1.0:1787696025.463673:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787696025.463677:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f6680 x1874531139563648/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.463685:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787696025.463686:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696025.463688:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787696025.463690:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696025.463692:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696025.463695:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139563648:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696025.463699:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787696025.463701:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787696025.463702:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696025.463704:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787696025.463707:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787696025.463709:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696025.463713:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787696025.463715:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787696025.463718:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787696025.463721:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139563648, portal 10 00000100:00000001:1.0:1787696025.463722:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787696025.463723:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930153600 : -128555779398016 : ffff8b14466f6680) 00000100:00000040:1.0:1787696025.463728:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f6680 x1874531139563648/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696050 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.463733:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787696025.463734:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787696025.463736:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787696025.463737:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139563648, offset 0 00000400:00000200:1.0:1787696025.463740:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787696025.463748:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696025.463751:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787696025.463753:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787696025.463755:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787696025.463757:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787696025.463758:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787696025.463760:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17213:17213:256:4294967295] 192.168.201.140@tcp LPNI seq info [17212:17212:8:4294967295] 00000400:00000200:1.0:1787696025.463763:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696025.463768:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787696025.463771:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696025.463773:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787696025.463776:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787696025.463778:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1787696025.463781:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787696025.463783:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1787696025.463786:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696025.463787:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787696025.463789:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787696025.463792:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787696025.463794:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787696025.463795:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787696025.463797:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787696025.463800:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f6680 x1874531139563648/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696050 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.463815:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696025.463818:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f6680 x1874531139563648/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696050 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.463829:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696025.463830:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787696025.463834:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696025.463839:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: using a timeout of 1476791 usecs (1477 jiffies) 00000400:00000200:1.0:1787696025.463842:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696025.463850:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298587078 00000100:00000001:1.0:1787696025.463854:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298597177 00000100:00000001:1.0:1787696025.463856:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298587078 00000100:00000001:1.0:1787696025.463858:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298597177 00000100:00000001:1.0:1787696025.463860:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696025.463862:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696025.463864:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787696025.463868:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696025/real 1787696025] req@ffff8b14466f6680 x1874531139563648/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787696025.463876:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787696025.463878:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787696025.463879:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787696025.463882:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787696025.463883:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787696025.463886:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787696025.463889:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f6680 x1874531139563648/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787696025.463896:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f6680 x1874531139563648/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.463910:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787696025.463912:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.463913:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696025.463915:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787696025.463919:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f6680 x1874531139563648/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.463926:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787696025.463930:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f6680 x1874531139563648/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.463938:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787696025.463946:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.463948:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696025.463949:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.463951:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787696025.463953:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787696025.463955:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1442fe1000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787696025.463959:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787696025.463962:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787696025.463966:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f6680 x1874531139563648/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787696025.463972:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139563648:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696025.463975:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787696025.463977:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5880 x1874531139563776/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.463981:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787696025.463982:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696025.463997:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787696025.463999:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696025.464000:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696025.464002:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139563776:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696025.464005:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787696025.464006:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787696025.464007:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696025.464009:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787696025.464017:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547400. 02000000:00000001:1.0:1787696025.464018:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696025.464023:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787696025.464025:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787696025.464028:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787696025.464030:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139563776, portal 10 00000100:00000001:1.0:1787696025.464035:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787696025.464036:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930150016 : -128555779401600 : ffff8b14466f5880) 00000100:00000040:1.0:1787696025.464041:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5880 x1874531139563776/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696050 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464046:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787696025.464048:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787696025.464049:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787696025.464051:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139563776, offset 0 00000400:00000200:1.0:1787696025.464053:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787696025.464056:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696025.464059:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787696025.464061:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787696025.464063:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787696025.464065:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787696025.464067:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787696025.464069:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17214:17214:256:4294967295] 192.168.201.140@tcp LPNI seq info [17213:17213:8:4294967295] 00000400:00000200:1.0:1787696025.464071:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696025.464076:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787696025.464079:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696025.464080:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787696025.464083:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787696025.464085:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1787696025.464088:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787696025.464090:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1787696025.464093:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696025.464094:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787696025.464096:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787696025.464099:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787696025.464101:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787696025.464102:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787696025.464104:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787696025.464107:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5880 x1874531139563776/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696050 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464128:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696025.464132:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5880 x1874531139563776/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696050 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464139:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696025.464140:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787696025.464145:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696025.464148:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: using a timeout of 324506 usecs (325 jiffies) 00000400:00000200:1.0:1787696025.464152:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696025.464154:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298585926 00000100:00000001:1.0:1787696025.464157:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298597177 00000100:00000001:1.0:1787696025.464160:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298585926 00000100:00000001:1.0:1787696025.464162:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298597177 00000100:00000001:1.0:1787696025.464164:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696025.464166:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696025.464168:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787696025.464172:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696025/real 1787696025] req@ffff8b14466f5880 x1874531139563776/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787696025.464180:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787696025.464181:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787696025.464183:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787696025.464185:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787696025.464186:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787696025.464187:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787696025.464190:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5880 x1874531139563776/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787696025.464195:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5880 x1874531139563776/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464200:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787696025.464205:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.464207:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696025.464208:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787696025.464210:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5880 x1874531139563776/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464215:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787696025.464217:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5880 x1874531139563776/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464222:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787696025.464223:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.464224:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696025.464225:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.464226:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787696025.464227:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787696025.464229:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458081800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787696025.464231:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787696025.464233:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787696025.464235:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5880 x1874531139563776/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787696025.464240:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139563776:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696025.464242:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787696025.464244:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4000 x1874531139563904/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464248:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787696025.464249:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696025.464250:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787696025.464252:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696025.464253:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696025.464258:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139563904:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787696025.464260:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787696025.464261:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787696025.464263:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696025.464264:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787696025.464266:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545c00. 02000000:00000001:1.0:1787696025.464267:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696025.464270:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787696025.464271:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787696025.464274:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787696025.464276:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139563904, portal 4 00000100:00000001:1.0:1787696025.464277:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787696025.464278:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930143744 : -128555779407872 : ffff8b14466f4000) 00000100:00000040:1.0:1787696025.464283:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4000 x1874531139563904/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696050 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464287:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787696025.464288:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787696025.464290:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787696025.464291:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139563904, offset 0 00000400:00000200:1.0:1787696025.464294:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787696025.464296:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696025.464299:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787696025.464301:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787696025.464302:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787696025.464304:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787696025.464306:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787696025.464308:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17215:17215:256:4294967295] 192.168.201.140@tcp LPNI seq info [17214:17214:8:4294967295] 00000400:00000200:1.0:1787696025.464310:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696025.464315:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787696025.464318:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696025.464320:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787696025.464326:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787696025.464328:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1787696025.464330:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787696025.464332:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1787696025.464335:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696025.464337:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787696025.464339:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787696025.464341:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787696025.464343:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787696025.464344:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787696025.464346:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787696025.464348:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4000 x1874531139563904/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696050 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464354:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696025.464356:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4000 x1874531139563904/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696050 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464360:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696025.464361:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787696025.464383:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696025.464388:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: using a timeout of 354733 usecs (355 jiffies) 00000400:00000200:1.0:1787696025.464392:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696025.464394:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298585957 00000100:00000001:1.0:1787696025.464397:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298597177 00000100:00000001:1.0:1787696025.464399:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298585957 00000100:00000001:1.0:1787696025.464402:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298597177 00000100:00000001:1.0:1787696025.464404:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696025.464406:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696025.464408:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787696025.464412:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696025/real 1787696025] req@ffff8b14466f4000 x1874531139563904/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787696025.464420:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787696025.464427:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787696025.464429:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787696025.464430:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787696025.464432:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787696025.464434:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787696025.464436:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4000 x1874531139563904/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787696025.464442:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4000 x1874531139563904/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464446:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787696025.464447:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.464448:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696025.464449:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787696025.464451:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4000 x1874531139563904/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464456:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787696025.464458:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4000 x1874531139563904/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464463:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787696025.464464:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.464465:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696025.464466:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.464467:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787696025.464468:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787696025.464470:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458083000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787696025.464472:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787696025.464473:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787696025.464475:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4000 x1874531139563904/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787696025.464486:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139563904:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787696025.464489:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.464491:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696025.464493:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4a80 x1874531139563520/t0(0) o250->MGC192.168.201.140@tcp@192.168.201.140@tcp:26/25 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464497:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787696025.464499:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787696025.464501:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446546800. 02000000:00000001:1.0:1787696025.464504:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787696025.464505:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787696025.464506:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe5000 refcount=3 obd=MGC192.168.201.140@tcp 00000020:00000001:1.0:1787696025.464508:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787696025.464510:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675400. 02000000:00000001:1.0:1787696025.464513:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787696025.464514:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787696025.464515:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4a80. 00000100:00000001:1.0:1787696025.464517:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787696025.464518:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.464519:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696025.464522:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f6680 x1874531139563648/t0(0) o38->lustre-MDT0000-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464526:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787696025.464526:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787696025.464527:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787696025.464529:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787696025.464530:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787696025.464531:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1442fe1000 refcount=7 obd=lustre-MDT0000-mdc-ffff8b1458183800 00000020:00000001:1.0:1787696025.464532:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787696025.464533:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677400. 02000000:00000001:1.0:1787696025.464535:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787696025.464536:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787696025.464537:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f6680. 00000100:00000001:1.0:1787696025.464541:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787696025.464542:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.464544:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696025.464546:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5880 x1874531139563776/t0(0) o38->lustre-MDT0001-mdc-ffff8b1458183800@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464550:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787696025.464551:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787696025.464552:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547400. 02000000:00000001:1.0:1787696025.464554:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787696025.464554:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787696025.464555:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458081800 refcount=7 obd=lustre-MDT0001-mdc-ffff8b1458183800 00000020:00000001:1.0:1787696025.464556:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787696025.464557:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446675800. 02000000:00000001:1.0:1787696025.464558:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787696025.464559:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787696025.464560:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5880. 00000100:00000001:1.0:1787696025.464561:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787696025.464562:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.464563:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696025.464564:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4000 x1874531139563904/t0(0) o8->lustre-OST0000-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464568:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787696025.464569:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787696025.464570:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545c00. 02000000:00000001:1.0:1787696025.464571:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787696025.464571:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787696025.464572:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458083000 refcount=2 obd=lustre-OST0000-osc-ffff8b1458183800 00000020:00000001:1.0:1787696025.464573:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787696025.464574:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676400. 02000000:00000001:1.0:1787696025.464576:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787696025.464577:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787696025.464577:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4000. 00000100:00000001:1.0:1787696025.464578:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787696025.464579:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.464581:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.464586:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787696025.464591:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696025.464593:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787696025.464594:0:2402:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787696025.464595:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787696025.464598:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f4700 x1874531139564032/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464602:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787696025.464603:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696025.464605:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787696025.464606:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696025.464607:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696025.464609:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139564032:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787696025.464611:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787696025.464613:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787696025.464614:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696025.464615:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787696025.464617:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545c00. 02000000:00000001:1.0:1787696025.464618:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696025.464620:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787696025.464622:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787696025.464624:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787696025.464626:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139564032, portal 4 00000100:00000001:1.0:1787696025.464628:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787696025.464629:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930145536 : -128555779406080 : ffff8b14466f4700) 00000100:00000040:1.0:1787696025.464633:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f4700 x1874531139564032/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696050 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464637:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787696025.464639:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787696025.464640:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787696025.464642:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1874531139564032, offset 0 00000400:00000200:1.0:1787696025.464644:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787696025.464647:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696025.464649:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787696025.464654:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787696025.464656:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787696025.464658:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787696025.464660:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787696025.464662:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17216:17216:256:4294967295] 192.168.201.140@tcp LPNI seq info [17215:17215:8:4294967295] 00000400:00000200:1.0:1787696025.464665:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696025.464670:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787696025.464672:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696025.464675:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787696025.464677:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787696025.464679:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1787696025.464682:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787696025.464683:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1787696025.464686:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696025.464687:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787696025.464689:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787696025.464691:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787696025.464693:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787696025.464695:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787696025.464696:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787696025.464699:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f4700 x1874531139564032/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696050 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464704:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696025.464705:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f4700 x1874531139564032/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 0 dl 1787696050 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464710:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696025.464711:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787696025.464713:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696025.464715:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: using a timeout of 26856 usecs (27 jiffies) 00000400:00000200:1.0:1787696025.464717:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696025.464719:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 2 is 4298585629 00000100:00000001:1.0:1787696025.464720:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 2, keep current deadline 4298597177 00000100:00000001:1.0:1787696025.464725:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS new deadline estimation for cpu 3 is 4298585629 00000100:00000001:1.0:1787696025.464727:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff8b1458183800: PM QoS request active for cpu 3, keep current deadline 4298597177 00000100:00000001:1.0:1787696025.464728:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696025.464729:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696025.464731:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787696025.464733:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696025/real 1787696025] req@ffff8b14466f4700 x1874531139564032/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787696025.464739:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787696025.464740:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787696025.464741:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787696025.464742:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787696025.464744:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787696025.464745:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787696025.464747:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f4700 x1874531139564032/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787696025.464752:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f4700 x1874531139564032/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464756:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787696025.464757:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.464758:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696025.464759:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787696025.464761:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f4700 x1874531139564032/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464765:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787696025.464767:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f4700 x1874531139564032/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464771:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787696025.464773:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.464774:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696025.464777:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.464778:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787696025.464779:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787696025.464782:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b1458080800 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787696025.464784:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787696025.464785:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787696025.464788:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f4700 x1874531139564032/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787696025.464792:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:33b90c3f-84c2-4e9c-a496-7dca4143e2a1:2402:1874531139564032:192.168.201.140@tcp:8:kworker.0 00000100:00000001:1.0:1787696025.464794:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787696025.464796:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5500 x1874531139564160/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464800:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787696025.464801:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696025.464802:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787696025.464804:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696025.464805:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696025.464807:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139564160:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696025.464809:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787696025.464810:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787696025.464811:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696025.464813:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787696025.464814:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446547400. 02000000:00000001:1.0:1787696025.464815:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696025.464818:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787696025.464819:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787696025.464821:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787696025.464823:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139564160, portal 10 00000100:00000001:1.0:1787696025.464824:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787696025.464825:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930149120 : -128555779402496 : ffff8b14466f5500) 00000100:00000040:1.0:1787696025.464832:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5500 x1874531139564160/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696050 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464837:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787696025.464838:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787696025.464839:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787696025.464841:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139564160, offset 0 00000400:00000200:1.0:1787696025.464844:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787696025.464846:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696025.464848:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787696025.464850:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787696025.464851:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787696025.464853:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787696025.464855:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787696025.464857:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17217:17217:256:4294967295] 192.168.201.140@tcp LPNI seq info [17216:17216:8:4294967295] 00000400:00000200:1.0:1787696025.464860:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696025.464864:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787696025.464866:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696025.464868:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787696025.464870:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787696025.464872:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1787696025.464875:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787696025.464877:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1787696025.464880:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696025.464881:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787696025.464883:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787696025.464885:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787696025.464887:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787696025.464888:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787696025.464890:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787696025.464892:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5500 x1874531139564160/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696050 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464897:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696025.464902:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5500 x1874531139564160/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696050 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464907:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696025.464908:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787696025.464910:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696025.464912:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: using a timeout of 212250 usecs (213 jiffies) 00000400:00000200:1.0:1787696025.464914:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696025.464916:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298585815 00000100:00000001:1.0:1787696025.464917:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298597177 00000100:00000001:1.0:1787696025.464919:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298585815 00000100:00000001:1.0:1787696025.464920:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298597177 00000100:00000001:1.0:1787696025.464921:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696025.464922:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696025.464923:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787696025.464926:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696025/real 1787696025] req@ffff8b14466f5500 x1874531139564160/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787696025.464930:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787696025.464931:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787696025.464933:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787696025.464934:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787696025.464935:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787696025.464936:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787696025.464938:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5500 x1874531139564160/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787696025.464944:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5500 x1874531139564160/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464948:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787696025.464949:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.464950:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696025.464950:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787696025.464956:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5500 x1874531139564160/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464960:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787696025.464963:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5500 x1874531139564160/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.464967:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787696025.464968:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.464969:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696025.464970:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.464971:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787696025.464972:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787696025.464973:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515e000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787696025.464976:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787696025.464977:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787696025.464979:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5500 x1874531139564160/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787696025.464998:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139564160:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696025.465001:0:2402:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1787696025.465003:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff8b14466f5f80 x1874531139564288/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.465008:0:2402:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787696025.465009:0:2402:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696025.465010:0:2402:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787696025.465012:0:2402:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787696025.465013:0:2402:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787696025.465015:0:2402:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139564288:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696025.465016:0:2402:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787696025.465022:0:2402:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787696025.465024:0:2402:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787696025.465025:0:2402:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1787696025.465027:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787696025.465028:0:2402:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1787696025.465031:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000010:1.0:1787696025.465032:0:2402:0:(lib-me.c:74:LNetMEAttach()) slab-alloced 'me': 88 at ffff8b14547021b8. 00000400:00000010:1.0:1787696025.465035:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f462a8. 00000100:00000200:1.0:1787696025.465037:0:2402:0:(niobuf.c:1097:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1874531139564288, portal 10 00000100:00000001:1.0:1787696025.465038:0:2402:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787696025.465039:0:2402:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446615517930151808 : -128555779399808 : ffff8b14466f5f80) 00000100:00000040:1.0:1787696025.465044:0:2402:0:(niobuf.c:1117:ptl_send_rpc()) @@@ send flags=200 req@ffff8b14466f5f80 x1874531139564288/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696050 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.465048:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1787696025.465050:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.201.140@tcp 00000400:00000010:1.0:1787696025.465051:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff8b1442f46dd0. 00000100:00000200:1.0:1787696025.465052:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1874531139564288, offset 0 00000400:00000200:1.0:1787696025.465055:0:2402:0:(lib-move.c:5157:LNetPut()) LNetPut -> 12345-192.168.201.140@tcp 00000400:00000200:1.0:1787696025.465057:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696025.465059:0:2402:0:(lib-move.c:2813:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.201.140@tcp local destination 00000400:00000200:1.0:1787696025.465061:0:2402:0:(lib-move.c:2444:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1787696025.465063:0:2402:0:(lib-move.c:1625:lnet_get_best_ni()) selected best_ni 192.168.201.40@tcp 00000400:00000200:1.0:1787696025.465065:0:2402:0:(lib-move.c:1178:lnet_select_peer_ni()) 192.168.201.40@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1787696025.465066:0:2402:0:(lib-move.c:1247:lnet_select_peer_ni()) sd_best_lpni = 192.168.201.140@tcp 00000400:00000200:1.0:1787696025.465068:0:2402:0:(lib-move.c:1745:lnet_handle_send()) 192.168.201.40@tcp NI seq info: [17218:17218:256:4294967295] 192.168.201.140@tcp LPNI seq info [17217:17217:8:4294967295] 00000400:00000200:1.0:1787696025.465071:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000400:00000200:1.0:1787696025.465075:0:2402:0:(lib-move.c:1861:lnet_handle_send()) TRACE: 192.168.201.40@tcp(192.168.201.40@tcp:) -> 192.168.201.140@tcp(192.168.201.140@tcp:192.168.201.140@tcp) : PUT try# 0 00000800:00000200:1.0:1787696025.465078:0:2402:0:(socklnd_cb.c:931:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696025.465080:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff8b14460da300. 00000800:00000200:1.0:1787696025.465083:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.0:1787696025.465085:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000800:00000200:1.1:1787696025.465088:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff8b1460732a80] -> 12345-192.168.201.140@tcp (2) 00000800:00000200:1.1:1787696025.465090:0:2402:0:(socklnd_cb.c:778:ksocknal_find_connectable_conn_cb_locked()) Too soon to retry route 192.168.201.140 (cnted 0, interval 60, 42 secs later) 00000800:00000100:1.0:1787696025.465092:0:2402:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.201.140@tcp 00000800:00000010:1.0:1787696025.465097:0:2402:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff8b14460da300. 00000400:00000200:1.0:1787696025.465099:0:2402:0:(lib-msg.c:1121:lnet_is_health_check()) health check = 1, status = -5, hstatus = 7 00000400:00000200:1.0:1787696025.465101:0:2402:0:(lib-msg.c:895:lnet_health_check()) health check: 192.168.201.40@tcp->192.168.201.140@tcp: PUT: REMOTE_ERROR 00000400:00000200:1.0:1787696025.465103:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f46dd0 00000400:00000010:1.0:1787696025.465104:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f46dd0. 00000100:00000001:1.0:1787696025.465106:0:2402:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1787696025.465108:0:2402:0:(events.c:39:request_out_callback()) @@@ type 5, status -5 req@ffff8b14466f5f80 x1874531139564288/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696050 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.465117:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696025.465120:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff8b14466f5f80 x1874531139564288/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 0 dl 1787696050 ref 2 fl Rpc:eNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.465127:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696025.465128:0:2402:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787696025.465132:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1787696025.465135:0:2402:0:(niobuf.c:635:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: using a timeout of 190148 usecs (191 jiffies) 00000400:00000200:1.0:1787696025.465138:0:2402:0:(api-ni.c:1613:lnet_nid4_cpt_hash()) Match nid 192.168.201.140@tcp to cpt 1 00000100:00000001:1.0:1787696025.465141:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 2 is 4298585793 00000100:00000001:1.0:1787696025.465143:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 2, keep current deadline 4298597177 00000100:00000001:1.0:1787696025.465145:0:2402:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS new deadline estimation for cpu 3 is 4298585793 00000100:00000001:1.0:1787696025.465148:0:2402:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8b145807d000: PM QoS request active for cpu 3, keep current deadline 4298597177 00000100:00000001:1.0:1787696025.465150:0:2402:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787696025.465152:0:2402:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696025.465154:0:2402:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787696025.465158:0:2402:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787696025/real 1787696025] req@ffff8b14466f5f80 x1874531139564288/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787696025.465167:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787696025.465168:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1787696025.465170:0:2402:0:(lib-me.c:117:lnet_me_unlink()) slab-freed 'me': 88 at ffff8b14547021b8. 00000400:00000200:1.0:1787696025.465172:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff8b1442f462a8 00000400:00000010:1.0:1787696025.465174:0:2402:0:(lib-lnet.h:283:lnet_md_free()) slab-freed 'md': 136 at ffff8b1442f462a8. 00000100:00000001:1.0:1787696025.465182:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1787696025.465185:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff8b14466f5f80 x1874531139564288/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1787696025.465191:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff8b14466f5f80 x1874531139564288/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.465195:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787696025.465196:0:2402:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.465197:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696025.465198:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787696025.465200:0:2402:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8b14466f5f80 x1874531139564288/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.465205:0:2402:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1787696025.465207:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff8b14466f5f80 x1874531139564288/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Rpc:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.465211:0:2402:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787696025.465213:0:2402:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.465214:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787696025.465214:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.465216:0:2402:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787696025.465217:0:2402:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787696025.465218:0:2402:0:(import.c:65:import_set_state_nolock()) ffff8b144515c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787696025.465220:0:2402:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.201.140@tcp failed (-110) 00000100:00000001:1.0:1787696025.465221:0:2402:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:1.0:1787696025.465224:0:2402:0:(lustre_net.h:2405:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff8b14466f5f80 x1874531139564288/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Interpret:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787696025.465228:0:2402:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8b14466f5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:608c526c-eb6e-4a54-80a5-8969828241eb:2402:1874531139564288:192.168.201.140@tcp:38:kworker.0 00000100:00000001:1.0:1787696025.465231:0:2402:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.465235:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696025.465237:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f4700 x1874531139564032/t0(0) o8->lustre-OST0001-osc-ffff8b1458183800@192.168.201.140@tcp:28/4 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.465242:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787696025.465243:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787696025.465244:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545c00. 02000000:00000001:1.0:1787696025.465246:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787696025.465247:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787696025.465248:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b1458080800 refcount=2 obd=lustre-OST0001-osc-ffff8b1458183800 00000020:00000001:1.0:1787696025.465249:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787696025.465250:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446674000. 02000000:00000001:1.0:1787696025.465253:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787696025.465253:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787696025.465254:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f4700. 00000100:00000001:1.0:1787696025.465256:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787696025.465257:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.465258:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696025.465260:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5500 x1874531139564160/t0(0) o38->lustre-MDT0000-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.465264:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787696025.465265:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787696025.465265:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446547400. 02000000:00000001:1.0:1787696025.465267:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787696025.465267:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787696025.465268:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515e000 refcount=5 obd=lustre-MDT0000-mdc-ffff8b145807d000 00000020:00000001:1.0:1787696025.465269:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787696025.465270:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446676000. 02000000:00000001:1.0:1787696025.465271:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787696025.465272:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787696025.465273:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5500. 00000100:00000001:1.0:1787696025.465274:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787696025.465275:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.465276:0:2402:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1787696025.465278:0:2402:0:(client.c:2875:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff8b14466f5f80 x1874531139564288/t0(0) o38->lustre-MDT0001-mdc-ffff8b145807d000@192.168.201.140@tcp:12/10 lens 520/544 e 0 to 1 dl 1787696050 ref 1 fl Complete:EeXNQU/200/ffffffff rc -110/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787696025.465289:0:2402:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787696025.465290:0:2402:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1787696025.465291:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff8b1446545400. 02000000:00000001:1.0:1787696025.465292:0:2402:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787696025.465293:0:2402:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000040:1.0:1787696025.465293:0:2402:0:(genops.c:1223:class_import_put()) import ffff8b144515c000 refcount=8 obd=lustre-MDT0001-mdc-ffff8b145807d000 00000020:00000001:1.0:1787696025.465294:0:2402:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000010:1.0:1787696025.465295:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff8b1446677c00. 02000000:00000001:1.0:1787696025.465297:0:2402:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787696025.465297:0:2402:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1787696025.465298:0:2402:0:(client.c:612:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff8b14466f5f80. 00000100:00000001:1.0:1787696025.465300:0:2402:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787696025.465300:0:2402:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.465302:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787696025.465304:0:2402:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787696025.465305:0:2402:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787696025.465306:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787696025.465307:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787696025.526494:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787696025.526502:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787696025.526504:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787696025.526511:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:0.0:1787696025.526515:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787696025.526519:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696025.526522:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787696025.526524:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696025.526527:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696026.166461:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696026.166467:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696026.166471:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696026.166472:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696026.166475:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696026.166478:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787696026.166479:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787696026.166484:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787696026.166485:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787696026.166487:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696026.166489:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787696026.166491:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696026.166493:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696026.166494:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696026.166497:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696026.166498:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787696026.166499:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787696026.166500:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787696026.166501:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787696026.166518:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696026.166520:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787696026.166522:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696026.166524:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696026.166526:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696026.166530:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696026.166532:0:47390:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696026.166535:0:47390:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1787696026.166537:0:47390:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1787696026.166538:0:47390:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1787696026.166541:0:47390:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1787696026.166542:0:47390:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1787696026.166544:0:47390:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696026.166545:0:47390:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1787696026.166547:0:47390:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696026.166549:0:47390:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1787696026.166551:0:47390:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787696026.166594:0:39513:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787696026.166597:0:39513:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787696026.166600:0:39513:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787696026.550471:0:52523:0:(client.c:2567:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1787696026.550479:0:52523:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787696026.550482:0:52523:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787696026.550487:0:52523:0:(client.c:2715:ptlrpc_set_wait()) set ffff8b1460d9b6c0 going to sleep for 0 seconds 00000100:00000001:0.0:1787696026.550490:0:52523:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787696026.550493:0:52523:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696026.550495:0:52523:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787696026.550497:0:52523:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787696026.550499:0:52523:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) Debug log: 64579 lines, 64579 kept, 0 dropped, 0 bad.